@uploadista/core 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/.turbo/turbo-check.log +231 -0
- package/.turbo/turbo-format.log +5 -0
- package/LICENSE +21 -0
- package/README.md +1120 -0
- package/dist/chunk-CUT6urMc.cjs +1 -0
- package/dist/debounce-C2SeqcxD.js +2 -0
- package/dist/debounce-C2SeqcxD.js.map +1 -0
- package/dist/debounce-LZK7yS7Z.cjs +1 -0
- package/dist/errors/index.cjs +1 -0
- package/dist/errors/index.d.cts +3 -0
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.d.ts.map +1 -0
- package/dist/errors/index.js +2 -0
- package/dist/errors/uploadista-error.d.ts +209 -0
- package/dist/errors/uploadista-error.d.ts.map +1 -0
- package/dist/errors/uploadista-error.js +322 -0
- package/dist/errors-8i_aMxOE.js +1 -0
- package/dist/errors-CRm1FHHT.cjs +0 -0
- package/dist/flow/edge.d.ts +47 -0
- package/dist/flow/edge.d.ts.map +1 -0
- package/dist/flow/edge.js +40 -0
- package/dist/flow/event.d.ts +206 -0
- package/dist/flow/event.d.ts.map +1 -0
- package/dist/flow/event.js +53 -0
- package/dist/flow/flow-server.d.ts +223 -0
- package/dist/flow/flow-server.d.ts.map +1 -0
- package/dist/flow/flow-server.js +614 -0
- package/dist/flow/flow.d.ts +238 -0
- package/dist/flow/flow.d.ts.map +1 -0
- package/dist/flow/flow.js +629 -0
- package/dist/flow/index.cjs +1 -0
- package/dist/flow/index.d.cts +6 -0
- package/dist/flow/index.d.ts +24 -0
- package/dist/flow/index.d.ts.map +1 -0
- package/dist/flow/index.js +24 -0
- package/dist/flow/node.d.ts +136 -0
- package/dist/flow/node.d.ts.map +1 -0
- package/dist/flow/node.js +153 -0
- package/dist/flow/nodes/index.d.ts +8 -0
- package/dist/flow/nodes/index.d.ts.map +1 -0
- package/dist/flow/nodes/index.js +7 -0
- package/dist/flow/nodes/input-node.d.ts +78 -0
- package/dist/flow/nodes/input-node.d.ts.map +1 -0
- package/dist/flow/nodes/input-node.js +233 -0
- package/dist/flow/nodes/storage-node.d.ts +67 -0
- package/dist/flow/nodes/storage-node.d.ts.map +1 -0
- package/dist/flow/nodes/storage-node.js +94 -0
- package/dist/flow/nodes/streaming-input-node.d.ts +69 -0
- package/dist/flow/nodes/streaming-input-node.d.ts.map +1 -0
- package/dist/flow/nodes/streaming-input-node.js +156 -0
- package/dist/flow/nodes/transform-node.d.ts +85 -0
- package/dist/flow/nodes/transform-node.d.ts.map +1 -0
- package/dist/flow/nodes/transform-node.js +107 -0
- package/dist/flow/parallel-scheduler.d.ts +175 -0
- package/dist/flow/parallel-scheduler.d.ts.map +1 -0
- package/dist/flow/parallel-scheduler.js +193 -0
- package/dist/flow/plugins/credential-provider.d.ts +47 -0
- package/dist/flow/plugins/credential-provider.d.ts.map +1 -0
- package/dist/flow/plugins/credential-provider.js +24 -0
- package/dist/flow/plugins/image-ai-plugin.d.ts +61 -0
- package/dist/flow/plugins/image-ai-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/image-ai-plugin.js +21 -0
- package/dist/flow/plugins/image-plugin.d.ts +52 -0
- package/dist/flow/plugins/image-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/image-plugin.js +22 -0
- package/dist/flow/plugins/types/describe-image-node.d.ts +16 -0
- package/dist/flow/plugins/types/describe-image-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/describe-image-node.js +9 -0
- package/dist/flow/plugins/types/index.d.ts +9 -0
- package/dist/flow/plugins/types/index.d.ts.map +1 -0
- package/dist/flow/plugins/types/index.js +8 -0
- package/dist/flow/plugins/types/optimize-node.d.ts +20 -0
- package/dist/flow/plugins/types/optimize-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/optimize-node.js +11 -0
- package/dist/flow/plugins/types/remove-background-node.d.ts +16 -0
- package/dist/flow/plugins/types/remove-background-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/remove-background-node.js +9 -0
- package/dist/flow/plugins/types/resize-node.d.ts +21 -0
- package/dist/flow/plugins/types/resize-node.d.ts.map +1 -0
- package/dist/flow/plugins/types/resize-node.js +16 -0
- package/dist/flow/plugins/zip-plugin.d.ts +62 -0
- package/dist/flow/plugins/zip-plugin.d.ts.map +1 -0
- package/dist/flow/plugins/zip-plugin.js +21 -0
- package/dist/flow/typed-flow.d.ts +90 -0
- package/dist/flow/typed-flow.d.ts.map +1 -0
- package/dist/flow/typed-flow.js +59 -0
- package/dist/flow/types/flow-file.d.ts +45 -0
- package/dist/flow/types/flow-file.d.ts.map +1 -0
- package/dist/flow/types/flow-file.js +27 -0
- package/dist/flow/types/flow-job.d.ts +118 -0
- package/dist/flow/types/flow-job.d.ts.map +1 -0
- package/dist/flow/types/flow-job.js +11 -0
- package/dist/flow/types/flow-types.d.ts +321 -0
- package/dist/flow/types/flow-types.d.ts.map +1 -0
- package/dist/flow/types/flow-types.js +52 -0
- package/dist/flow/types/index.d.ts +4 -0
- package/dist/flow/types/index.d.ts.map +1 -0
- package/dist/flow/types/index.js +3 -0
- package/dist/flow/types/run-args.d.ts +38 -0
- package/dist/flow/types/run-args.d.ts.map +1 -0
- package/dist/flow/types/run-args.js +30 -0
- package/dist/flow/types/type-validator.d.ts +26 -0
- package/dist/flow/types/type-validator.d.ts.map +1 -0
- package/dist/flow/types/type-validator.js +134 -0
- package/dist/flow/utils/resolve-upload-metadata.d.ts +11 -0
- package/dist/flow/utils/resolve-upload-metadata.d.ts.map +1 -0
- package/dist/flow/utils/resolve-upload-metadata.js +28 -0
- package/dist/flow-2zXnEiWL.cjs +1 -0
- package/dist/flow-CRaKy7Vj.js +2 -0
- package/dist/flow-CRaKy7Vj.js.map +1 -0
- package/dist/generate-id-Dm-Vboxq.d.ts +34 -0
- package/dist/generate-id-Dm-Vboxq.d.ts.map +1 -0
- package/dist/generate-id-LjJRLD6N.d.cts +34 -0
- package/dist/generate-id-LjJRLD6N.d.cts.map +1 -0
- package/dist/generate-id-xHp_Z7Cl.cjs +1 -0
- package/dist/generate-id-yohS1ZDk.js +2 -0
- package/dist/generate-id-yohS1ZDk.js.map +1 -0
- package/dist/index-BO8GZlbD.d.cts +1040 -0
- package/dist/index-BO8GZlbD.d.cts.map +1 -0
- package/dist/index-BoGG5KAY.d.ts +1 -0
- package/dist/index-BtBZHVmz.d.cts +1 -0
- package/dist/index-D-CoVpkZ.d.ts +1004 -0
- package/dist/index-D-CoVpkZ.d.ts.map +1 -0
- package/dist/index.cjs +1 -0
- package/dist/index.d.cts +6 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/logger/logger.cjs +1 -0
- package/dist/logger/logger.d.cts +8 -0
- package/dist/logger/logger.d.cts.map +1 -0
- package/dist/logger/logger.d.ts +5 -0
- package/dist/logger/logger.d.ts.map +1 -0
- package/dist/logger/logger.js +10 -0
- package/dist/logger/logger.js.map +1 -0
- package/dist/semaphore-0ZwjVpyF.js +2 -0
- package/dist/semaphore-0ZwjVpyF.js.map +1 -0
- package/dist/semaphore-BHprIjFI.d.cts +37 -0
- package/dist/semaphore-BHprIjFI.d.cts.map +1 -0
- package/dist/semaphore-DThupBkc.d.ts +37 -0
- package/dist/semaphore-DThupBkc.d.ts.map +1 -0
- package/dist/semaphore-DVrONiAV.cjs +1 -0
- package/dist/stream-limiter-CoWKv39w.js +2 -0
- package/dist/stream-limiter-CoWKv39w.js.map +1 -0
- package/dist/stream-limiter-JgOwmkMa.cjs +1 -0
- package/dist/streams/multi-stream.cjs +1 -0
- package/dist/streams/multi-stream.d.cts +91 -0
- package/dist/streams/multi-stream.d.cts.map +1 -0
- package/dist/streams/multi-stream.d.ts +86 -0
- package/dist/streams/multi-stream.d.ts.map +1 -0
- package/dist/streams/multi-stream.js +149 -0
- package/dist/streams/multi-stream.js.map +1 -0
- package/dist/streams/stream-limiter.cjs +1 -0
- package/dist/streams/stream-limiter.d.cts +36 -0
- package/dist/streams/stream-limiter.d.cts.map +1 -0
- package/dist/streams/stream-limiter.d.ts +27 -0
- package/dist/streams/stream-limiter.d.ts.map +1 -0
- package/dist/streams/stream-limiter.js +49 -0
- package/dist/streams/stream-splitter.cjs +1 -0
- package/dist/streams/stream-splitter.d.cts +68 -0
- package/dist/streams/stream-splitter.d.cts.map +1 -0
- package/dist/streams/stream-splitter.d.ts +51 -0
- package/dist/streams/stream-splitter.d.ts.map +1 -0
- package/dist/streams/stream-splitter.js +175 -0
- package/dist/streams/stream-splitter.js.map +1 -0
- package/dist/types/data-store-registry.d.ts +13 -0
- package/dist/types/data-store-registry.d.ts.map +1 -0
- package/dist/types/data-store-registry.js +4 -0
- package/dist/types/data-store.d.ts +316 -0
- package/dist/types/data-store.d.ts.map +1 -0
- package/dist/types/data-store.js +157 -0
- package/dist/types/event-broadcaster.d.ts +28 -0
- package/dist/types/event-broadcaster.d.ts.map +1 -0
- package/dist/types/event-broadcaster.js +6 -0
- package/dist/types/event-emitter.d.ts +378 -0
- package/dist/types/event-emitter.d.ts.map +1 -0
- package/dist/types/event-emitter.js +223 -0
- package/dist/types/index.cjs +1 -0
- package/dist/types/index.d.cts +6 -0
- package/dist/types/index.d.ts +10 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +9 -0
- package/dist/types/input-file.d.ts +104 -0
- package/dist/types/input-file.d.ts.map +1 -0
- package/dist/types/input-file.js +27 -0
- package/dist/types/kv-store.d.ts +281 -0
- package/dist/types/kv-store.d.ts.map +1 -0
- package/dist/types/kv-store.js +234 -0
- package/dist/types/middleware.d.ts +17 -0
- package/dist/types/middleware.d.ts.map +1 -0
- package/dist/types/middleware.js +21 -0
- package/dist/types/upload-event.d.ts +105 -0
- package/dist/types/upload-event.d.ts.map +1 -0
- package/dist/types/upload-event.js +71 -0
- package/dist/types/upload-file.d.ts +136 -0
- package/dist/types/upload-file.d.ts.map +1 -0
- package/dist/types/upload-file.js +34 -0
- package/dist/types/websocket.d.ts +144 -0
- package/dist/types/websocket.d.ts.map +1 -0
- package/dist/types/websocket.js +40 -0
- package/dist/types-BT-cvi7T.cjs +1 -0
- package/dist/types-DhU2j-XF.js +2 -0
- package/dist/types-DhU2j-XF.js.map +1 -0
- package/dist/upload/convert-to-stream.d.ts +38 -0
- package/dist/upload/convert-to-stream.d.ts.map +1 -0
- package/dist/upload/convert-to-stream.js +43 -0
- package/dist/upload/convert-upload-to-flow-file.d.ts +14 -0
- package/dist/upload/convert-upload-to-flow-file.d.ts.map +1 -0
- package/dist/upload/convert-upload-to-flow-file.js +21 -0
- package/dist/upload/create-upload.d.ts +68 -0
- package/dist/upload/create-upload.d.ts.map +1 -0
- package/dist/upload/create-upload.js +157 -0
- package/dist/upload/index.cjs +1 -0
- package/dist/upload/index.d.cts +6 -0
- package/dist/upload/index.d.ts +4 -0
- package/dist/upload/index.d.ts.map +1 -0
- package/dist/upload/index.js +3 -0
- package/dist/upload/mime.d.ts +24 -0
- package/dist/upload/mime.d.ts.map +1 -0
- package/dist/upload/mime.js +351 -0
- package/dist/upload/upload-chunk.d.ts +58 -0
- package/dist/upload/upload-chunk.d.ts.map +1 -0
- package/dist/upload/upload-chunk.js +277 -0
- package/dist/upload/upload-server.d.ts +221 -0
- package/dist/upload/upload-server.d.ts.map +1 -0
- package/dist/upload/upload-server.js +181 -0
- package/dist/upload/upload-strategy-negotiator.d.ts +148 -0
- package/dist/upload/upload-strategy-negotiator.d.ts.map +1 -0
- package/dist/upload/upload-strategy-negotiator.js +217 -0
- package/dist/upload/upload-url.d.ts +68 -0
- package/dist/upload/upload-url.d.ts.map +1 -0
- package/dist/upload/upload-url.js +142 -0
- package/dist/upload/write-to-store.d.ts +77 -0
- package/dist/upload/write-to-store.d.ts.map +1 -0
- package/dist/upload/write-to-store.js +147 -0
- package/dist/upload-DLuICjpP.cjs +1 -0
- package/dist/upload-DaXO34dE.js +2 -0
- package/dist/upload-DaXO34dE.js.map +1 -0
- package/dist/uploadista-error-BB-Wdiz9.cjs +22 -0
- package/dist/uploadista-error-BVsVxqvz.js +23 -0
- package/dist/uploadista-error-BVsVxqvz.js.map +1 -0
- package/dist/uploadista-error-CwxYs4EB.d.ts +52 -0
- package/dist/uploadista-error-CwxYs4EB.d.ts.map +1 -0
- package/dist/uploadista-error-kKlhLRhY.d.cts +52 -0
- package/dist/uploadista-error-kKlhLRhY.d.cts.map +1 -0
- package/dist/utils/checksum.d.ts +22 -0
- package/dist/utils/checksum.d.ts.map +1 -0
- package/dist/utils/checksum.js +49 -0
- package/dist/utils/debounce.cjs +1 -0
- package/dist/utils/debounce.d.cts +38 -0
- package/dist/utils/debounce.d.cts.map +1 -0
- package/dist/utils/debounce.d.ts +36 -0
- package/dist/utils/debounce.d.ts.map +1 -0
- package/dist/utils/debounce.js +73 -0
- package/dist/utils/generate-id.cjs +1 -0
- package/dist/utils/generate-id.d.cts +2 -0
- package/dist/utils/generate-id.d.ts +32 -0
- package/dist/utils/generate-id.d.ts.map +1 -0
- package/dist/utils/generate-id.js +23 -0
- package/dist/utils/md5.cjs +1 -0
- package/dist/utils/md5.d.cts +73 -0
- package/dist/utils/md5.d.cts.map +1 -0
- package/dist/utils/md5.d.ts +71 -0
- package/dist/utils/md5.d.ts.map +1 -0
- package/dist/utils/md5.js +417 -0
- package/dist/utils/md5.js.map +1 -0
- package/dist/utils/once.cjs +1 -0
- package/dist/utils/once.d.cts +25 -0
- package/dist/utils/once.d.cts.map +1 -0
- package/dist/utils/once.d.ts +21 -0
- package/dist/utils/once.d.ts.map +1 -0
- package/dist/utils/once.js +54 -0
- package/dist/utils/once.js.map +1 -0
- package/dist/utils/semaphore.cjs +1 -0
- package/dist/utils/semaphore.d.cts +3 -0
- package/dist/utils/semaphore.d.ts +78 -0
- package/dist/utils/semaphore.d.ts.map +1 -0
- package/dist/utils/semaphore.js +134 -0
- package/dist/utils/throttle.cjs +1 -0
- package/dist/utils/throttle.d.cts +24 -0
- package/dist/utils/throttle.d.cts.map +1 -0
- package/dist/utils/throttle.d.ts +18 -0
- package/dist/utils/throttle.d.ts.map +1 -0
- package/dist/utils/throttle.js +20 -0
- package/dist/utils/throttle.js.map +1 -0
- package/docs/PARALLEL_EXECUTION.md +206 -0
- package/docs/PARALLEL_EXECUTION_QUICKSTART.md +142 -0
- package/docs/PARALLEL_EXECUTION_REFACTOR.md +184 -0
- package/package.json +80 -0
- package/src/errors/__tests__/uploadista-error.test.ts +251 -0
- package/src/errors/index.ts +2 -0
- package/src/errors/uploadista-error.ts +394 -0
- package/src/flow/README.md +352 -0
- package/src/flow/edge.test.ts +146 -0
- package/src/flow/edge.ts +60 -0
- package/src/flow/event.ts +229 -0
- package/src/flow/flow-server.ts +1089 -0
- package/src/flow/flow.ts +1050 -0
- package/src/flow/index.ts +28 -0
- package/src/flow/node.ts +249 -0
- package/src/flow/nodes/index.ts +8 -0
- package/src/flow/nodes/input-node.ts +296 -0
- package/src/flow/nodes/storage-node.ts +128 -0
- package/src/flow/nodes/transform-node.ts +154 -0
- package/src/flow/parallel-scheduler.ts +259 -0
- package/src/flow/plugins/credential-provider.ts +48 -0
- package/src/flow/plugins/image-ai-plugin.ts +66 -0
- package/src/flow/plugins/image-plugin.ts +60 -0
- package/src/flow/plugins/types/describe-image-node.ts +16 -0
- package/src/flow/plugins/types/index.ts +9 -0
- package/src/flow/plugins/types/optimize-node.ts +18 -0
- package/src/flow/plugins/types/remove-background-node.ts +18 -0
- package/src/flow/plugins/types/resize-node.ts +26 -0
- package/src/flow/plugins/zip-plugin.ts +69 -0
- package/src/flow/typed-flow.ts +279 -0
- package/src/flow/types/flow-file.ts +51 -0
- package/src/flow/types/flow-job.ts +138 -0
- package/src/flow/types/flow-types.ts +353 -0
- package/src/flow/types/index.ts +6 -0
- package/src/flow/types/run-args.ts +40 -0
- package/src/flow/types/type-validator.ts +204 -0
- package/src/flow/utils/resolve-upload-metadata.ts +48 -0
- package/src/index.ts +5 -0
- package/src/logger/logger.ts +14 -0
- package/src/streams/stream-limiter.test.ts +150 -0
- package/src/streams/stream-limiter.ts +75 -0
- package/src/types/data-store.ts +427 -0
- package/src/types/event-broadcaster.ts +39 -0
- package/src/types/event-emitter.ts +349 -0
- package/src/types/index.ts +9 -0
- package/src/types/input-file.ts +107 -0
- package/src/types/kv-store.ts +375 -0
- package/src/types/middleware.ts +54 -0
- package/src/types/upload-event.ts +75 -0
- package/src/types/upload-file.ts +139 -0
- package/src/types/websocket.ts +65 -0
- package/src/upload/convert-to-stream.ts +48 -0
- package/src/upload/create-upload.ts +214 -0
- package/src/upload/index.ts +3 -0
- package/src/upload/mime.ts +436 -0
- package/src/upload/upload-chunk.ts +364 -0
- package/src/upload/upload-server.ts +390 -0
- package/src/upload/upload-strategy-negotiator.ts +316 -0
- package/src/upload/upload-url.ts +173 -0
- package/src/upload/write-to-store.ts +211 -0
- package/src/utils/checksum.ts +61 -0
- package/src/utils/debounce.test.ts +126 -0
- package/src/utils/debounce.ts +89 -0
- package/src/utils/generate-id.ts +35 -0
- package/src/utils/md5.ts +475 -0
- package/src/utils/once.test.ts +83 -0
- package/src/utils/once.ts +63 -0
- package/src/utils/throttle.test.ts +101 -0
- package/src/utils/throttle.ts +29 -0
- package/tsconfig.json +20 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/tsdown.config.ts +25 -0
- package/vitest.config.ts +15 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parallel execution scheduler for flow nodes.
|
|
3
|
+
*
|
|
4
|
+
* The ParallelScheduler analyzes flow dependencies and groups nodes into execution
|
|
5
|
+
* levels where nodes at the same level can run in parallel. It manages concurrency
|
|
6
|
+
* using Effect's built-in concurrency control to prevent resource exhaustion.
|
|
7
|
+
*
|
|
8
|
+
* @module flow/parallel-scheduler
|
|
9
|
+
* @see {@link ParallelScheduler} for the main scheduler class
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
* This scheduler groups nodes by execution level (respecting dependencies) and executes
|
|
13
|
+
* each level in parallel with controlled concurrency. Levels are executed sequentially
|
|
14
|
+
* to ensure dependencies are satisfied before dependent nodes execute.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```typescript
|
|
18
|
+
* const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
|
|
19
|
+
*
|
|
20
|
+
* // Group nodes by execution level
|
|
21
|
+
* const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
|
|
22
|
+
*
|
|
23
|
+
* // Execute nodes in a level with Effect
|
|
24
|
+
* const results = yield* scheduler.executeNodesInParallel([
|
|
25
|
+
* () => executeNode("node1"),
|
|
26
|
+
* () => executeNode("node2"),
|
|
27
|
+
* () => executeNode("node3")
|
|
28
|
+
* ]);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
import { Effect } from "effect";
|
|
32
|
+
import type { FlowNode } from "./types/flow-types";
|
|
33
|
+
/**
|
|
34
|
+
* Represents a level in the execution hierarchy where all nodes can run in parallel.
|
|
35
|
+
*
|
|
36
|
+
* @property level - The execution level (0 = first to execute, higher = later)
|
|
37
|
+
* @property nodes - Array of node IDs that can execute in parallel at this level
|
|
38
|
+
*
|
|
39
|
+
* @example
|
|
40
|
+
* ```
|
|
41
|
+
* Level 0: [input_node] (no dependencies)
|
|
42
|
+
* Level 1: [resize, optimize] (all depend on level 0)
|
|
43
|
+
* Level 2: [storage] (depends on level 1)
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export interface ExecutionLevel {
|
|
47
|
+
level: number;
|
|
48
|
+
nodes: string[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Configuration options for the ParallelScheduler.
|
|
52
|
+
*
|
|
53
|
+
* @property maxConcurrency - Maximum number of nodes to execute in parallel (default: 4)
|
|
54
|
+
* Controls how many nodes run simultaneously within a level
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* ```typescript
|
|
58
|
+
* const scheduler = new ParallelScheduler({ maxConcurrency: 8 });
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
export interface ParallelSchedulerConfig {
|
|
62
|
+
maxConcurrency?: number;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Scheduler for executing flow nodes in parallel while respecting dependencies.
|
|
66
|
+
*
|
|
67
|
+
* The scheduler performs topological sorting to identify nodes that can run
|
|
68
|
+
* concurrently, groups them into execution levels, and provides methods to
|
|
69
|
+
* execute them with controlled concurrency using Effect.
|
|
70
|
+
*
|
|
71
|
+
* Key responsibilities:
|
|
72
|
+
* - Analyze flow dependencies and detect cycles
|
|
73
|
+
* - Group nodes into parallel execution levels
|
|
74
|
+
* - Execute levels in parallel with concurrency limits
|
|
75
|
+
* - Provide utilities to check parallel execution feasibility
|
|
76
|
+
*/
|
|
77
|
+
export declare class ParallelScheduler {
|
|
78
|
+
private maxConcurrency;
|
|
79
|
+
/**
|
|
80
|
+
* Creates a new ParallelScheduler instance.
|
|
81
|
+
*
|
|
82
|
+
* @param config - Configuration for the scheduler
|
|
83
|
+
* @example
|
|
84
|
+
* ```typescript
|
|
85
|
+
* const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
|
|
86
|
+
* ```
|
|
87
|
+
*/
|
|
88
|
+
constructor(config?: ParallelSchedulerConfig);
|
|
89
|
+
/**
|
|
90
|
+
* Groups nodes into execution levels where nodes in the same level can run in parallel.
|
|
91
|
+
*
|
|
92
|
+
* Uses Kahn's algorithm to perform topological sorting with level identification.
|
|
93
|
+
* Nodes are grouped by their distance from source nodes (input nodes with no dependencies).
|
|
94
|
+
*
|
|
95
|
+
* @param nodes - Array of flow nodes to analyze
|
|
96
|
+
* @param edges - Array of edges defining dependencies between nodes
|
|
97
|
+
* @returns Array of execution levels, ordered from 0 (no dependencies) onwards
|
|
98
|
+
* @throws Error if a cycle is detected in the flow graph
|
|
99
|
+
*
|
|
100
|
+
* @example
|
|
101
|
+
* ```typescript
|
|
102
|
+
* const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
|
|
103
|
+
* // levels = [
|
|
104
|
+
* // { level: 0, nodes: ['input_1'] },
|
|
105
|
+
* // { level: 1, nodes: ['resize_1', 'optimize_1'] },
|
|
106
|
+
* // { level: 2, nodes: ['output_1'] }
|
|
107
|
+
* // ]
|
|
108
|
+
* ```
|
|
109
|
+
*/
|
|
110
|
+
groupNodesByExecutionLevel(nodes: FlowNode<unknown, unknown>[], edges: Array<{
|
|
111
|
+
source: string;
|
|
112
|
+
target: string;
|
|
113
|
+
}>): ExecutionLevel[];
|
|
114
|
+
/**
|
|
115
|
+
* Executes a batch of Effect-based node executors in parallel with concurrency control.
|
|
116
|
+
*
|
|
117
|
+
* All executors are run in parallel, but the number of concurrent executions is limited
|
|
118
|
+
* by maxConcurrency. This prevents resource exhaustion while maximizing parallelism.
|
|
119
|
+
*
|
|
120
|
+
* @template T - The return type of each executor
|
|
121
|
+
* @template E - The error type of the Effects
|
|
122
|
+
* @template R - The requirements type of the Effects
|
|
123
|
+
*
|
|
124
|
+
* @param nodeExecutors - Array of Effect-returning functions to execute in parallel
|
|
125
|
+
* @returns Effect that resolves to array of results in the same order as input
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```typescript
|
|
129
|
+
* const results = yield* scheduler.executeNodesInParallel([
|
|
130
|
+
* () => executeNode("node1"),
|
|
131
|
+
* () => executeNode("node2"),
|
|
132
|
+
* () => executeNode("node3")
|
|
133
|
+
* ]);
|
|
134
|
+
* // results will be in order: [result1, result2, result3]
|
|
135
|
+
* ```
|
|
136
|
+
*/
|
|
137
|
+
executeNodesInParallel<T, E, R>(nodeExecutors: Array<() => Effect.Effect<T, E, R>>): Effect.Effect<T[], E, R>;
|
|
138
|
+
/**
|
|
139
|
+
* Determines if a set of nodes can be safely executed in parallel.
|
|
140
|
+
*
|
|
141
|
+
* Nodes can execute in parallel if all their dependencies have been completed.
|
|
142
|
+
* This is typically called to verify that nodes in an execution level are ready
|
|
143
|
+
* to run given the current node results.
|
|
144
|
+
*
|
|
145
|
+
* @param nodeIds - Array of node IDs to check
|
|
146
|
+
* @param nodeResults - Map of completed node IDs to their results
|
|
147
|
+
* @param reverseGraph - Dependency graph mapping node IDs to their incoming dependencies
|
|
148
|
+
* @returns true if all dependencies for all nodes are in nodeResults, false otherwise
|
|
149
|
+
*
|
|
150
|
+
* @example
|
|
151
|
+
* ```typescript
|
|
152
|
+
* const canRun = scheduler.canExecuteInParallel(
|
|
153
|
+
* ['resize_1', 'optimize_1'],
|
|
154
|
+
* nodeResults,
|
|
155
|
+
* reverseGraph
|
|
156
|
+
* );
|
|
157
|
+
* ```
|
|
158
|
+
*/
|
|
159
|
+
canExecuteInParallel(nodeIds: string[], nodeResults: Map<string, unknown>, reverseGraph: Record<string, string[]>): boolean;
|
|
160
|
+
/**
|
|
161
|
+
* Gets execution statistics for monitoring and debugging.
|
|
162
|
+
*
|
|
163
|
+
* @returns Object containing current scheduler configuration
|
|
164
|
+
*
|
|
165
|
+
* @example
|
|
166
|
+
* ```typescript
|
|
167
|
+
* const stats = scheduler.getStats();
|
|
168
|
+
* console.log(`Max concurrency: ${stats.maxConcurrency}`);
|
|
169
|
+
* ```
|
|
170
|
+
*/
|
|
171
|
+
getStats(): {
|
|
172
|
+
maxConcurrency: number;
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=parallel-scheduler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-scheduler.d.ts","sourceRoot":"","sources":["../../src/flow/parallel-scheduler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,uBAAuB;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,cAAc,CAAS;IAE/B;;;;;;;;OAQG;gBACS,MAAM,GAAE,uBAA4B;IAIhD;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,0BAA0B,CACxB,KAAK,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EACnC,KAAK,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,GAC/C,cAAc,EAAE;IAwDnB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,sBAAsB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAC5B,aAAa,EAAE,KAAK,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GACjD,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAS3B;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,oBAAoB,CAClB,OAAO,EAAE,MAAM,EAAE,EACjB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,EACjC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GACrC,OAAO;IAOV;;;;;;;;;;OAUG;IACH,QAAQ;;;CAKT"}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parallel execution scheduler for flow nodes.
|
|
3
|
+
*
|
|
4
|
+
* The ParallelScheduler analyzes flow dependencies and groups nodes into execution
|
|
5
|
+
* levels where nodes at the same level can run in parallel. It manages concurrency
|
|
6
|
+
* using Effect's built-in concurrency control to prevent resource exhaustion.
|
|
7
|
+
*
|
|
8
|
+
* @module flow/parallel-scheduler
|
|
9
|
+
* @see {@link ParallelScheduler} for the main scheduler class
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
* This scheduler groups nodes by execution level (respecting dependencies) and executes
|
|
13
|
+
* each level in parallel with controlled concurrency. Levels are executed sequentially
|
|
14
|
+
* to ensure dependencies are satisfied before dependent nodes execute.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```typescript
|
|
18
|
+
* const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
|
|
19
|
+
*
|
|
20
|
+
* // Group nodes by execution level
|
|
21
|
+
* const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
|
|
22
|
+
*
|
|
23
|
+
* // Execute nodes in a level with Effect
|
|
24
|
+
* const results = yield* scheduler.executeNodesInParallel([
|
|
25
|
+
* () => executeNode("node1"),
|
|
26
|
+
* () => executeNode("node2"),
|
|
27
|
+
* () => executeNode("node3")
|
|
28
|
+
* ]);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
import { Effect } from "effect";
|
|
32
|
+
/**
|
|
33
|
+
* Scheduler for executing flow nodes in parallel while respecting dependencies.
|
|
34
|
+
*
|
|
35
|
+
* The scheduler performs topological sorting to identify nodes that can run
|
|
36
|
+
* concurrently, groups them into execution levels, and provides methods to
|
|
37
|
+
* execute them with controlled concurrency using Effect.
|
|
38
|
+
*
|
|
39
|
+
* Key responsibilities:
|
|
40
|
+
* - Analyze flow dependencies and detect cycles
|
|
41
|
+
* - Group nodes into parallel execution levels
|
|
42
|
+
* - Execute levels in parallel with concurrency limits
|
|
43
|
+
* - Provide utilities to check parallel execution feasibility
|
|
44
|
+
*/
|
|
45
|
+
export class ParallelScheduler {
|
|
46
|
+
maxConcurrency;
|
|
47
|
+
/**
|
|
48
|
+
* Creates a new ParallelScheduler instance.
|
|
49
|
+
*
|
|
50
|
+
* @param config - Configuration for the scheduler
|
|
51
|
+
* @example
|
|
52
|
+
* ```typescript
|
|
53
|
+
* const scheduler = new ParallelScheduler({ maxConcurrency: 4 });
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
constructor(config = {}) {
|
|
57
|
+
this.maxConcurrency = config.maxConcurrency ?? 4;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Groups nodes into execution levels where nodes in the same level can run in parallel.
|
|
61
|
+
*
|
|
62
|
+
* Uses Kahn's algorithm to perform topological sorting with level identification.
|
|
63
|
+
* Nodes are grouped by their distance from source nodes (input nodes with no dependencies).
|
|
64
|
+
*
|
|
65
|
+
* @param nodes - Array of flow nodes to analyze
|
|
66
|
+
* @param edges - Array of edges defining dependencies between nodes
|
|
67
|
+
* @returns Array of execution levels, ordered from 0 (no dependencies) onwards
|
|
68
|
+
* @throws Error if a cycle is detected in the flow graph
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* ```typescript
|
|
72
|
+
* const levels = scheduler.groupNodesByExecutionLevel(nodes, edges);
|
|
73
|
+
* // levels = [
|
|
74
|
+
* // { level: 0, nodes: ['input_1'] },
|
|
75
|
+
* // { level: 1, nodes: ['resize_1', 'optimize_1'] },
|
|
76
|
+
* // { level: 2, nodes: ['output_1'] }
|
|
77
|
+
* // ]
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
groupNodesByExecutionLevel(nodes, edges) {
|
|
81
|
+
// Build dependency graph
|
|
82
|
+
const graph = {};
|
|
83
|
+
const inDegree = {};
|
|
84
|
+
// Initialize graph structure
|
|
85
|
+
nodes.forEach((node) => {
|
|
86
|
+
graph[node.id] = [];
|
|
87
|
+
inDegree[node.id] = 0;
|
|
88
|
+
});
|
|
89
|
+
// Build edges and calculate in-degrees
|
|
90
|
+
edges.forEach((edge) => {
|
|
91
|
+
graph[edge.source]?.push(edge.target);
|
|
92
|
+
inDegree[edge.target] = (inDegree[edge.target] || 0) + 1;
|
|
93
|
+
});
|
|
94
|
+
const levels = [];
|
|
95
|
+
const processedNodes = new Set();
|
|
96
|
+
let levelIndex = 0;
|
|
97
|
+
// Use Kahn's algorithm to group nodes by level
|
|
98
|
+
while (processedNodes.size < nodes.length) {
|
|
99
|
+
// Find all nodes with zero in-degree that haven't been processed
|
|
100
|
+
const currentLevelNodes = Object.keys(inDegree).filter((nodeId) => inDegree[nodeId] === 0 && !processedNodes.has(nodeId));
|
|
101
|
+
if (currentLevelNodes.length === 0) {
|
|
102
|
+
throw new Error("Cycle detected in flow graph - cannot execute in parallel");
|
|
103
|
+
}
|
|
104
|
+
levels.push({
|
|
105
|
+
level: levelIndex++,
|
|
106
|
+
nodes: currentLevelNodes,
|
|
107
|
+
});
|
|
108
|
+
// Remove current level nodes and update in-degrees for dependent nodes
|
|
109
|
+
currentLevelNodes.forEach((nodeId) => {
|
|
110
|
+
processedNodes.add(nodeId);
|
|
111
|
+
delete inDegree[nodeId];
|
|
112
|
+
// Decrease in-degree for all nodes that depend on this node
|
|
113
|
+
graph[nodeId]?.forEach((dependentId) => {
|
|
114
|
+
if (inDegree[dependentId] !== undefined) {
|
|
115
|
+
inDegree[dependentId]--;
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
return levels;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Executes a batch of Effect-based node executors in parallel with concurrency control.
|
|
124
|
+
*
|
|
125
|
+
* All executors are run in parallel, but the number of concurrent executions is limited
|
|
126
|
+
* by maxConcurrency. This prevents resource exhaustion while maximizing parallelism.
|
|
127
|
+
*
|
|
128
|
+
* @template T - The return type of each executor
|
|
129
|
+
* @template E - The error type of the Effects
|
|
130
|
+
* @template R - The requirements type of the Effects
|
|
131
|
+
*
|
|
132
|
+
* @param nodeExecutors - Array of Effect-returning functions to execute in parallel
|
|
133
|
+
* @returns Effect that resolves to array of results in the same order as input
|
|
134
|
+
*
|
|
135
|
+
* @example
|
|
136
|
+
* ```typescript
|
|
137
|
+
* const results = yield* scheduler.executeNodesInParallel([
|
|
138
|
+
* () => executeNode("node1"),
|
|
139
|
+
* () => executeNode("node2"),
|
|
140
|
+
* () => executeNode("node3")
|
|
141
|
+
* ]);
|
|
142
|
+
* // results will be in order: [result1, result2, result3]
|
|
143
|
+
* ```
|
|
144
|
+
*/
|
|
145
|
+
executeNodesInParallel(nodeExecutors) {
|
|
146
|
+
return Effect.all(nodeExecutors.map((executor) => executor()), {
|
|
147
|
+
concurrency: this.maxConcurrency,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Determines if a set of nodes can be safely executed in parallel.
|
|
152
|
+
*
|
|
153
|
+
* Nodes can execute in parallel if all their dependencies have been completed.
|
|
154
|
+
* This is typically called to verify that nodes in an execution level are ready
|
|
155
|
+
* to run given the current node results.
|
|
156
|
+
*
|
|
157
|
+
* @param nodeIds - Array of node IDs to check
|
|
158
|
+
* @param nodeResults - Map of completed node IDs to their results
|
|
159
|
+
* @param reverseGraph - Dependency graph mapping node IDs to their incoming dependencies
|
|
160
|
+
* @returns true if all dependencies for all nodes are in nodeResults, false otherwise
|
|
161
|
+
*
|
|
162
|
+
* @example
|
|
163
|
+
* ```typescript
|
|
164
|
+
* const canRun = scheduler.canExecuteInParallel(
|
|
165
|
+
* ['resize_1', 'optimize_1'],
|
|
166
|
+
* nodeResults,
|
|
167
|
+
* reverseGraph
|
|
168
|
+
* );
|
|
169
|
+
* ```
|
|
170
|
+
*/
|
|
171
|
+
canExecuteInParallel(nodeIds, nodeResults, reverseGraph) {
|
|
172
|
+
return nodeIds.every((nodeId) => {
|
|
173
|
+
const dependencies = reverseGraph[nodeId] || [];
|
|
174
|
+
return dependencies.every((depId) => nodeResults.has(depId));
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Gets execution statistics for monitoring and debugging.
|
|
179
|
+
*
|
|
180
|
+
* @returns Object containing current scheduler configuration
|
|
181
|
+
*
|
|
182
|
+
* @example
|
|
183
|
+
* ```typescript
|
|
184
|
+
* const stats = scheduler.getStats();
|
|
185
|
+
* console.log(`Max concurrency: ${stats.maxConcurrency}`);
|
|
186
|
+
* ```
|
|
187
|
+
*/
|
|
188
|
+
getStats() {
|
|
189
|
+
return {
|
|
190
|
+
maxConcurrency: this.maxConcurrency,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { Context, type Effect } from "effect";
|
|
2
|
+
import type { UploadistaError } from "@/errors";
|
|
3
|
+
/**
|
|
4
|
+
* Shape definition for the Credential Provider interface.
|
|
5
|
+
* Defines the contract for retrieving credentials for various services.
|
|
6
|
+
*/
|
|
7
|
+
export interface CredentialProviderShape {
|
|
8
|
+
/**
|
|
9
|
+
* Retrieves credentials for a specific service and client.
|
|
10
|
+
*
|
|
11
|
+
* @param params - Parameters for credential retrieval
|
|
12
|
+
* @param params.clientId - Unique identifier for the client, or null if not available
|
|
13
|
+
* @param params.serviceType - Optional service type to get specific credentials for
|
|
14
|
+
* @returns An Effect that resolves to a record of credential key-value pairs
|
|
15
|
+
* @throws {UploadistaError} When credential retrieval fails
|
|
16
|
+
*/
|
|
17
|
+
getCredential: (params: {
|
|
18
|
+
clientId: string | null;
|
|
19
|
+
serviceType?: string;
|
|
20
|
+
}) => Effect.Effect<Record<string, unknown>, UploadistaError>;
|
|
21
|
+
}
|
|
22
|
+
declare const CredentialProvider_base: Context.TagClass<CredentialProvider, "CredentialProvider", CredentialProviderShape>;
|
|
23
|
+
/**
|
|
24
|
+
* Context tag for the Credential Provider.
|
|
25
|
+
*
|
|
26
|
+
* This tag provides a type-safe way to access credential functionality
|
|
27
|
+
* throughout the application using Effect's dependency injection system.
|
|
28
|
+
*
|
|
29
|
+
* @example
|
|
30
|
+
* ```typescript
|
|
31
|
+
* import { CredentialProvider } from "@uploadista/core/flow/plugins";
|
|
32
|
+
*
|
|
33
|
+
* // In your flow node
|
|
34
|
+
* const program = Effect.gen(function* () {
|
|
35
|
+
* const credentialProvider = yield* CredentialProvider;
|
|
36
|
+
* const credentials = yield* credentialProvider.getCredential({
|
|
37
|
+
* clientId: "user123",
|
|
38
|
+
* serviceType: "replicate"
|
|
39
|
+
* });
|
|
40
|
+
* return credentials;
|
|
41
|
+
* });
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
44
|
+
export declare class CredentialProvider extends CredentialProvider_base {
|
|
45
|
+
}
|
|
46
|
+
export {};
|
|
47
|
+
//# sourceMappingURL=credential-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"credential-provider.d.ts","sourceRoot":"","sources":["../../../src/flow/plugins/credential-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD;;;GAGG;AACH,MAAM,WAAW,uBAAuB;IACtC;;;;;;;;OAQG;IACH,aAAa,EAAE,CAAC,MAAM,EAAE;QACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC;CAC/D;;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,kBAAmB,SAAQ,uBAGrC;CAAG"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Context } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Context tag for the Credential Provider.
|
|
4
|
+
*
|
|
5
|
+
* This tag provides a type-safe way to access credential functionality
|
|
6
|
+
* throughout the application using Effect's dependency injection system.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { CredentialProvider } from "@uploadista/core/flow/plugins";
|
|
11
|
+
*
|
|
12
|
+
* // In your flow node
|
|
13
|
+
* const program = Effect.gen(function* () {
|
|
14
|
+
* const credentialProvider = yield* CredentialProvider;
|
|
15
|
+
* const credentials = yield* credentialProvider.getCredential({
|
|
16
|
+
* clientId: "user123",
|
|
17
|
+
* serviceType: "replicate"
|
|
18
|
+
* });
|
|
19
|
+
* return credentials;
|
|
20
|
+
* });
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
export class CredentialProvider extends Context.Tag("CredentialProvider")() {
|
|
24
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { UploadistaError } from "@uploadista/core/errors";
|
|
2
|
+
import { Context, type Effect } from "effect";
|
|
3
|
+
/**
|
|
4
|
+
* Context information for AI image processing operations.
|
|
5
|
+
* Contains client identification for tracking and billing purposes.
|
|
6
|
+
*/
|
|
7
|
+
export type ImageAiContext = {
|
|
8
|
+
/** Unique identifier for the client making the request, or null if not available */
|
|
9
|
+
clientId: string | null;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Shape definition for the Image AI Plugin interface.
|
|
13
|
+
* Defines the contract that all image AI implementations must follow.
|
|
14
|
+
*/
|
|
15
|
+
export type ImageAiPluginShape = {
|
|
16
|
+
/**
|
|
17
|
+
* Removes the background from an image using AI processing.
|
|
18
|
+
*
|
|
19
|
+
* @param inputUrl - The URL of the input image to process
|
|
20
|
+
* @param context - Context information including client ID for tracking
|
|
21
|
+
* @returns An Effect that resolves to an object containing the output image URL
|
|
22
|
+
* @throws {UploadistaError} When the background removal fails
|
|
23
|
+
*/
|
|
24
|
+
removeBackground: (inputUrl: string, context: ImageAiContext) => Effect.Effect<{
|
|
25
|
+
outputUrl: string;
|
|
26
|
+
}, UploadistaError>;
|
|
27
|
+
/**
|
|
28
|
+
* Generates a textual description of an image using AI analysis.
|
|
29
|
+
*
|
|
30
|
+
* @param inputUrl - The URL of the input image to analyze
|
|
31
|
+
* @param context - Context information including client ID for tracking
|
|
32
|
+
* @returns An Effect that resolves to an object containing the image description
|
|
33
|
+
* @throws {UploadistaError} When the image analysis fails
|
|
34
|
+
*/
|
|
35
|
+
describeImage: (inputUrl: string, context: ImageAiContext) => Effect.Effect<{
|
|
36
|
+
description: string;
|
|
37
|
+
}, UploadistaError>;
|
|
38
|
+
};
|
|
39
|
+
declare const ImageAiPlugin_base: Context.TagClass<ImageAiPlugin, "ImageAiPlugin", ImageAiPluginShape>;
|
|
40
|
+
/**
|
|
41
|
+
* Context tag for the Image AI Plugin.
|
|
42
|
+
*
|
|
43
|
+
* This tag provides a type-safe way to access image AI functionality
|
|
44
|
+
* throughout the application using Effect's dependency injection system.
|
|
45
|
+
*
|
|
46
|
+
* @example
|
|
47
|
+
* ```typescript
|
|
48
|
+
* import { ImageAiPlugin } from "@uploadista/core/flow/plugins";
|
|
49
|
+
*
|
|
50
|
+
* // In your flow node
|
|
51
|
+
* const program = Effect.gen(function* () {
|
|
52
|
+
* const imageAi = yield* ImageAiPlugin;
|
|
53
|
+
* const result = yield* imageAi.removeBackground(imageUrl, { clientId: "user123" });
|
|
54
|
+
* return result.outputUrl;
|
|
55
|
+
* });
|
|
56
|
+
* ```
|
|
57
|
+
*/
|
|
58
|
+
export declare class ImageAiPlugin extends ImageAiPlugin_base {
|
|
59
|
+
}
|
|
60
|
+
export {};
|
|
61
|
+
//# sourceMappingURL=image-ai-plugin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"image-ai-plugin.d.ts","sourceRoot":"","sources":["../../../src/flow/plugins/image-ai-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE9C;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,oFAAoF;IACpF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;;;;;;OAOG;IACH,gBAAgB,EAAE,CAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,cAAc,KACpB,MAAM,CAAC,MAAM,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,EAAE,eAAe,CAAC,CAAC;IAE3D;;;;;;;OAOG;IACH,aAAa,EAAE,CACb,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,cAAc,KACpB,MAAM,CAAC,MAAM,CAAC;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,EAAE,eAAe,CAAC,CAAC;CAC9D,CAAC;;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAc,SAAQ,kBAGhC;CAAG"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Context } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Context tag for the Image AI Plugin.
|
|
4
|
+
*
|
|
5
|
+
* This tag provides a type-safe way to access image AI functionality
|
|
6
|
+
* throughout the application using Effect's dependency injection system.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { ImageAiPlugin } from "@uploadista/core/flow/plugins";
|
|
11
|
+
*
|
|
12
|
+
* // In your flow node
|
|
13
|
+
* const program = Effect.gen(function* () {
|
|
14
|
+
* const imageAi = yield* ImageAiPlugin;
|
|
15
|
+
* const result = yield* imageAi.removeBackground(imageUrl, { clientId: "user123" });
|
|
16
|
+
* return result.outputUrl;
|
|
17
|
+
* });
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
export class ImageAiPlugin extends Context.Tag("ImageAiPlugin")() {
|
|
21
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { UploadistaError } from "@uploadista/core/errors";
|
|
2
|
+
import { Context, type Effect } from "effect";
|
|
3
|
+
import type { OptimizeParams } from "./types/optimize-node";
|
|
4
|
+
import type { ResizeParams } from "./types/resize-node";
|
|
5
|
+
/**
|
|
6
|
+
* Shape definition for the Image Plugin interface.
|
|
7
|
+
* Defines the contract that all image processing implementations must follow.
|
|
8
|
+
*/
|
|
9
|
+
export type ImagePluginShape = {
|
|
10
|
+
/**
|
|
11
|
+
* Optimizes an image by adjusting quality and format.
|
|
12
|
+
*
|
|
13
|
+
* @param input - The input image as a Uint8Array
|
|
14
|
+
* @param options - Optimization parameters including quality and format
|
|
15
|
+
* @returns An Effect that resolves to the optimized image as a Uint8Array
|
|
16
|
+
* @throws {UploadistaError} When image optimization fails
|
|
17
|
+
*/
|
|
18
|
+
optimize: (input: Uint8Array, options: OptimizeParams) => Effect.Effect<Uint8Array, UploadistaError>;
|
|
19
|
+
/**
|
|
20
|
+
* Resizes an image to specified dimensions.
|
|
21
|
+
*
|
|
22
|
+
* @param input - The input image as a Uint8Array
|
|
23
|
+
* @param options - Resize parameters including width, height, and fit mode
|
|
24
|
+
* @returns An Effect that resolves to the resized image as a Uint8Array
|
|
25
|
+
* @throws {UploadistaError} When image resizing fails
|
|
26
|
+
*/
|
|
27
|
+
resize: (input: Uint8Array, options: ResizeParams) => Effect.Effect<Uint8Array, UploadistaError>;
|
|
28
|
+
};
|
|
29
|
+
declare const ImagePlugin_base: Context.TagClass<ImagePlugin, "ImagePlugin", ImagePluginShape>;
|
|
30
|
+
/**
|
|
31
|
+
* Context tag for the Image Plugin.
|
|
32
|
+
*
|
|
33
|
+
* This tag provides a type-safe way to access image processing functionality
|
|
34
|
+
* throughout the application using Effect's dependency injection system.
|
|
35
|
+
*
|
|
36
|
+
* @example
|
|
37
|
+
* ```typescript
|
|
38
|
+
* import { ImagePlugin } from "@uploadista/core/flow/plugins";
|
|
39
|
+
*
|
|
40
|
+
* // In your flow node
|
|
41
|
+
* const program = Effect.gen(function* () {
|
|
42
|
+
* const imagePlugin = yield* ImagePlugin;
|
|
43
|
+
* const optimized = yield* imagePlugin.optimize(imageData, { quality: 80, format: "webp" });
|
|
44
|
+
* const resized = yield* imagePlugin.resize(optimized, { width: 800, height: 600, fit: "cover" });
|
|
45
|
+
* return resized;
|
|
46
|
+
* });
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
export declare class ImagePlugin extends ImagePlugin_base {
|
|
50
|
+
}
|
|
51
|
+
export {};
|
|
52
|
+
//# sourceMappingURL=image-plugin.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"image-plugin.d.ts","sourceRoot":"","sources":["../../../src/flow/plugins/image-plugin.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;;;;;OAOG;IACH,QAAQ,EAAE,CACR,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,cAAc,KACpB,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IAEhD;;;;;;;OAOG;IACH,MAAM,EAAE,CACN,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,YAAY,KAClB,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;CACjD,CAAC;;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,WAAY,SAAQ,gBAG9B;CAAG"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { Context } from "effect";
|
|
2
|
+
/**
|
|
3
|
+
* Context tag for the Image Plugin.
|
|
4
|
+
*
|
|
5
|
+
* This tag provides a type-safe way to access image processing functionality
|
|
6
|
+
* throughout the application using Effect's dependency injection system.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { ImagePlugin } from "@uploadista/core/flow/plugins";
|
|
11
|
+
*
|
|
12
|
+
* // In your flow node
|
|
13
|
+
* const program = Effect.gen(function* () {
|
|
14
|
+
* const imagePlugin = yield* ImagePlugin;
|
|
15
|
+
* const optimized = yield* imagePlugin.optimize(imageData, { quality: 80, format: "webp" });
|
|
16
|
+
* const resized = yield* imagePlugin.resize(optimized, { width: 800, height: 600, fit: "cover" });
|
|
17
|
+
* return resized;
|
|
18
|
+
* });
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export class ImagePlugin extends Context.Tag("ImagePlugin")() {
|
|
22
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Zod schema for validating describe image node parameters.
|
|
4
|
+
* Defines the structure and validation rules for image description requests.
|
|
5
|
+
*/
|
|
6
|
+
export declare const describeImageParamsSchema: z.ZodObject<{
|
|
7
|
+
serviceType: z.ZodOptional<z.ZodEnum<{
|
|
8
|
+
replicate: "replicate";
|
|
9
|
+
}>>;
|
|
10
|
+
}, z.core.$strip>;
|
|
11
|
+
/**
|
|
12
|
+
* Parameters for the describe image node.
|
|
13
|
+
* Controls which AI service to use for generating image descriptions.
|
|
14
|
+
*/
|
|
15
|
+
export type DescribeImageParams = z.infer<typeof describeImageParamsSchema>;
|
|
16
|
+
//# sourceMappingURL=describe-image-node.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"describe-image-node.d.ts","sourceRoot":"","sources":["../../../../src/flow/plugins/types/describe-image-node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,yBAAyB;;;;iBAGpC,CAAC;AAEH;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Zod schema for validating describe image node parameters.
|
|
4
|
+
* Defines the structure and validation rules for image description requests.
|
|
5
|
+
*/
|
|
6
|
+
export const describeImageParamsSchema = z.object({
|
|
7
|
+
/** Optional service type to use for image description (currently supports "replicate") */
|
|
8
|
+
serviceType: z.enum(["replicate"]).optional(),
|
|
9
|
+
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Re-exports all type definitions for flow plugin nodes.
|
|
3
|
+
* This module provides a centralized way to import all plugin node types.
|
|
4
|
+
*/
|
|
5
|
+
export * from "./describe-image-node";
|
|
6
|
+
export * from "./optimize-node";
|
|
7
|
+
export * from "./remove-background-node";
|
|
8
|
+
export * from "./resize-node";
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/flow/plugins/types/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,cAAc,uBAAuB,CAAC;AACtC,cAAc,iBAAiB,CAAC;AAChC,cAAc,0BAA0B,CAAC;AACzC,cAAc,eAAe,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Re-exports all type definitions for flow plugin nodes.
|
|
3
|
+
* This module provides a centralized way to import all plugin node types.
|
|
4
|
+
*/
|
|
5
|
+
export * from "./describe-image-node";
|
|
6
|
+
export * from "./optimize-node";
|
|
7
|
+
export * from "./remove-background-node";
|
|
8
|
+
export * from "./resize-node";
|