@uploadista/core 0.0.16 → 0.0.17-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/flow/index.cjs +1 -1
- package/dist/flow/index.d.cts +2 -2
- package/dist/flow/index.d.mts +2 -2
- package/dist/flow/index.mjs +1 -1
- package/dist/flow-7dlRoq1h.cjs +1 -0
- package/dist/flow-l-iFJavi.mjs +2 -0
- package/dist/flow-l-iFJavi.mjs.map +1 -0
- package/dist/{index-DJ5ug-bj.d.mts → index-BTGpo7gz.d.mts} +1022 -1052
- package/dist/index-BTGpo7gz.d.mts.map +1 -0
- package/dist/{index-Dp6mTOB2.d.cts → index-D9YSNFdA.d.cts} +1022 -1052
- package/dist/index-D9YSNFdA.d.cts.map +1 -0
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +2 -2
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +1 -1
- package/dist/testing/index.cjs +3 -3
- package/dist/testing/index.d.cts +1 -1
- package/dist/testing/index.d.mts +1 -1
- package/dist/testing/index.mjs +3 -3
- package/dist/types/index.d.cts +1 -1
- package/dist/types/index.d.mts +1 -1
- package/dist/upload/index.d.cts +1 -1
- package/dist/upload/index.d.mts +1 -1
- package/package.json +3 -3
- package/src/flow/flow-server.ts +9 -7
- package/src/flow/flow.ts +117 -16
- package/src/flow/node-types/index.ts +61 -18
- package/src/flow/node.ts +3 -16
- package/src/flow/nodes/index.ts +0 -1
- package/src/flow/nodes/input-node.ts +0 -2
- package/src/flow/nodes/transform-node.ts +4 -0
- package/src/flow/type-guards.ts +79 -0
- package/src/flow/typed-flow.ts +29 -17
- package/src/flow/types/flow-types.ts +45 -18
- package/tests/flow/flow.test.ts +35 -48
- package/tests/flow/node.test.ts +4 -12
- package/tests/flow/type-system.test.ts +719 -769
- package/tests/types/typed-event-emitter.test.ts +26 -7
- package/type-tests/flow.test-d.ts +21 -16
- package/type-tests/type-utils.test-d.ts +3 -2
- package/dist/flow-BtI159Fi.mjs +0 -2
- package/dist/flow-BtI159Fi.mjs.map +0 -1
- package/dist/flow-DK3kacMQ.cjs +0 -1
- package/dist/index-DJ5ug-bj.d.mts.map +0 -1
- package/dist/index-Dp6mTOB2.d.cts.map +0 -1
- package/src/flow/nodes/storage-node.ts +0 -130
|
@@ -162,8 +162,6 @@ declare enum NodeType {
|
|
|
162
162
|
input = "input",
|
|
163
163
|
/** Transforms data during flow execution */
|
|
164
164
|
process = "process",
|
|
165
|
-
/** Saves data to storage backends */
|
|
166
|
-
output = "output",
|
|
167
165
|
/** Routes data based on conditions */
|
|
168
166
|
conditional = "conditional",
|
|
169
167
|
/** Splits data to multiple outputs */
|
|
@@ -201,7 +199,7 @@ type ConditionValue = string | number;
|
|
|
201
199
|
* @param config.id - Unique identifier for this node in the flow
|
|
202
200
|
* @param config.name - Human-readable name for the node
|
|
203
201
|
* @param config.description - Description of what this node does
|
|
204
|
-
* @param config.type - The type of node (input, process,
|
|
202
|
+
* @param config.type - The type of node (input, process, conditional, multiplex, merge)
|
|
205
203
|
* @param config.inputSchema - Zod schema for validating input data
|
|
206
204
|
* @param config.outputSchema - Zod schema for validating output data
|
|
207
205
|
* @param config.run - The processing function to execute for this node
|
|
@@ -213,7 +211,7 @@ type ConditionValue = string | number;
|
|
|
213
211
|
* @param config.retry.maxRetries - Maximum number of retry attempts (default: 0)
|
|
214
212
|
* @param config.retry.retryDelay - Base delay in milliseconds between retries (default: 1000)
|
|
215
213
|
* @param config.retry.exponentialBackoff - Whether to use exponential backoff for retries (default: true)
|
|
216
|
-
* @param config.nodeTypeId - Optional type ID from the registry (e.g., "storage-output-v1"). If provided, the node type must be registered
|
|
214
|
+
* @param config.nodeTypeId - Optional type ID from the registry (e.g., "storage-output-v1"). If provided, the node type must be registered.
|
|
217
215
|
*
|
|
218
216
|
* @returns An Effect that succeeds with the created FlowNode
|
|
219
217
|
*
|
|
@@ -597,11 +595,6 @@ type BuiltInTypedOutput = {
|
|
|
597
595
|
data: UploadFile;
|
|
598
596
|
nodeId: string;
|
|
599
597
|
timestamp: string;
|
|
600
|
-
} | {
|
|
601
|
-
nodeType: "streaming-input-v1";
|
|
602
|
-
data: UploadFile;
|
|
603
|
-
nodeId: string;
|
|
604
|
-
timestamp: string;
|
|
605
598
|
};
|
|
606
599
|
/**
|
|
607
600
|
* Custom typed output for user-defined node types.
|
|
@@ -637,7 +630,6 @@ type CustomTypedOutput<T = unknown> = {
|
|
|
637
630
|
* @remarks
|
|
638
631
|
* **Built-in types (automatic narrowing):**
|
|
639
632
|
* - `storage-output-v1` - Storage node output (UploadFile)
|
|
640
|
-
* - `streaming-input-v1` - Streaming input node (UploadFile)
|
|
641
633
|
*
|
|
642
634
|
* Use switch statements for automatic narrowing:
|
|
643
635
|
* ```typescript
|
|
@@ -647,10 +639,6 @@ type CustomTypedOutput<T = unknown> = {
|
|
|
647
639
|
* // ✅ output.data is automatically UploadFile
|
|
648
640
|
* console.log(output.data.url);
|
|
649
641
|
* break;
|
|
650
|
-
* case 'streaming-input-v1':
|
|
651
|
-
* // ✅ output.data is automatically UploadFile
|
|
652
|
-
* console.log(output.data.size);
|
|
653
|
-
* break;
|
|
654
642
|
* }
|
|
655
643
|
* }
|
|
656
644
|
* ```
|
|
@@ -991,6 +979,45 @@ type FlowConfig<TFlowInputSchema extends z.ZodSchema<any>, TFlowOutputSchema ext
|
|
|
991
979
|
enabled?: boolean;
|
|
992
980
|
maxConcurrency?: number;
|
|
993
981
|
};
|
|
982
|
+
hooks?: {
|
|
983
|
+
/**
|
|
984
|
+
* Called when a sink node (terminal node with no outgoing edges) produces an output.
|
|
985
|
+
* This hook runs after auto-persistence for UploadFile outputs.
|
|
986
|
+
*
|
|
987
|
+
* Use this hook to perform additional post-processing such as:
|
|
988
|
+
* - Saving output metadata to a database
|
|
989
|
+
* - Tracking outputs in external systems
|
|
990
|
+
* - Adding custom metadata to outputs
|
|
991
|
+
* - Triggering downstream workflows
|
|
992
|
+
*
|
|
993
|
+
* The hook receives the output and context, and can optionally modify
|
|
994
|
+
* and return the output (e.g., adding metadata fields).
|
|
995
|
+
*
|
|
996
|
+
* @param context - Output context including the output data, node ID, flow ID, etc.
|
|
997
|
+
* @returns Effect that resolves to the (optionally modified) output
|
|
998
|
+
*
|
|
999
|
+
* @example
|
|
1000
|
+
* ```typescript
|
|
1001
|
+
* hooks: {
|
|
1002
|
+
* onNodeOutput: ({ output, nodeId, flowId }) =>
|
|
1003
|
+
* Effect.gen(function* () {
|
|
1004
|
+
* // Save to database
|
|
1005
|
+
* yield* saveToDatabase(output);
|
|
1006
|
+
* // Return output with additional metadata
|
|
1007
|
+
* return { ...output, metadata: { ...output.metadata, tracked: true } };
|
|
1008
|
+
* })
|
|
1009
|
+
* }
|
|
1010
|
+
* ```
|
|
1011
|
+
*/
|
|
1012
|
+
onNodeOutput?: <TOutput>(context: {
|
|
1013
|
+
output: TOutput;
|
|
1014
|
+
nodeId: string;
|
|
1015
|
+
flowId: string;
|
|
1016
|
+
jobId: string;
|
|
1017
|
+
storageId: string;
|
|
1018
|
+
clientId: string | null;
|
|
1019
|
+
}) => Effect.Effect<TOutput, UploadistaError, any>;
|
|
1020
|
+
};
|
|
994
1021
|
};
|
|
995
1022
|
//#endregion
|
|
996
1023
|
//#region src/flow/edge.d.ts
|
|
@@ -1045,1101 +1072,1117 @@ declare function createFlowEdge({
|
|
|
1045
1072
|
targetPort?: string;
|
|
1046
1073
|
}): FlowEdge;
|
|
1047
1074
|
//#endregion
|
|
1048
|
-
//#region src/
|
|
1075
|
+
//#region src/types/kv-store.d.ts
|
|
1049
1076
|
/**
|
|
1050
|
-
*
|
|
1051
|
-
* Contains the minimal information needed to reconstruct a flow.
|
|
1077
|
+
* Base key-value store interface for raw string storage.
|
|
1052
1078
|
*
|
|
1053
|
-
*
|
|
1054
|
-
*
|
|
1055
|
-
* @property nodes - Array of node data (without execution logic)
|
|
1056
|
-
* @property edges - Connections between nodes defining data flow
|
|
1057
|
-
*/
|
|
1058
|
-
type FlowData = {
|
|
1059
|
-
id: string;
|
|
1060
|
-
name: string;
|
|
1061
|
-
nodes: FlowNodeData[];
|
|
1062
|
-
edges: FlowEdge[];
|
|
1063
|
-
};
|
|
1064
|
-
/**
|
|
1065
|
-
* Extracts serializable flow data from a Flow instance.
|
|
1066
|
-
* Useful for storing flow definitions or sending them over the network.
|
|
1079
|
+
* This is the low-level interface that storage adapters implement.
|
|
1080
|
+
* It stores raw string values without type safety or serialization.
|
|
1067
1081
|
*
|
|
1068
|
-
* @
|
|
1069
|
-
* @
|
|
1070
|
-
* @
|
|
1082
|
+
* @property get - Retrieves a value by key, returns null if not found
|
|
1083
|
+
* @property set - Stores a value with the given key
|
|
1084
|
+
* @property delete - Removes a value by key
|
|
1085
|
+
* @property list - Optional operation to list all keys with a given prefix
|
|
1071
1086
|
*
|
|
1072
1087
|
* @example
|
|
1073
1088
|
* ```typescript
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1089
|
+
* // Implement a BaseKvStore with Redis
|
|
1090
|
+
* const redisKvStore: BaseKvStore = {
|
|
1091
|
+
* get: (key) => Effect.tryPromise({
|
|
1092
|
+
* try: () => redis.get(key),
|
|
1093
|
+
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1094
|
+
* }),
|
|
1095
|
+
*
|
|
1096
|
+
* set: (key, value) => Effect.tryPromise({
|
|
1097
|
+
* try: () => redis.set(key, value),
|
|
1098
|
+
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1099
|
+
* }),
|
|
1100
|
+
*
|
|
1101
|
+
* delete: (key) => Effect.tryPromise({
|
|
1102
|
+
* try: () => redis.del(key),
|
|
1103
|
+
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1104
|
+
* }),
|
|
1105
|
+
*
|
|
1106
|
+
* list: (prefix) => Effect.tryPromise({
|
|
1107
|
+
* try: () => redis.keys(`${prefix}*`),
|
|
1108
|
+
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1109
|
+
* })
|
|
1110
|
+
* };
|
|
1077
1111
|
* ```
|
|
1078
1112
|
*/
|
|
1079
|
-
|
|
1113
|
+
interface BaseKvStore {
|
|
1114
|
+
readonly get: (key: string) => Effect.Effect<string | null, UploadistaError>;
|
|
1115
|
+
readonly set: (key: string, value: string) => Effect.Effect<void, UploadistaError>;
|
|
1116
|
+
readonly delete: (key: string) => Effect.Effect<void, UploadistaError>;
|
|
1117
|
+
readonly list?: (keyPrefix: string) => Effect.Effect<Array<string>, UploadistaError>;
|
|
1118
|
+
}
|
|
1080
1119
|
/**
|
|
1081
|
-
*
|
|
1120
|
+
* Type-safe key-value store interface with automatic serialization.
|
|
1082
1121
|
*
|
|
1083
|
-
*
|
|
1122
|
+
* This wraps a BaseKvStore and handles JSON serialization/deserialization
|
|
1123
|
+
* for a specific data type, providing type safety and eliminating the need
|
|
1124
|
+
* for manual JSON.stringify/parse calls.
|
|
1084
1125
|
*
|
|
1085
|
-
* @
|
|
1086
|
-
*
|
|
1087
|
-
*
|
|
1126
|
+
* @template TData - The type of data stored in this KV store
|
|
1127
|
+
*
|
|
1128
|
+
* @property get - Retrieves and deserializes a value, fails if not found
|
|
1129
|
+
* @property set - Serializes and stores a value
|
|
1130
|
+
* @property delete - Removes a value by key
|
|
1131
|
+
* @property list - Optional operation to list all keys (without prefix)
|
|
1088
1132
|
*
|
|
1089
1133
|
* @example
|
|
1090
1134
|
* ```typescript
|
|
1091
|
-
*
|
|
1135
|
+
* // Use a typed KV store
|
|
1136
|
+
* const uploadStore: KvStore<UploadFile> = new TypedKvStore(
|
|
1137
|
+
* baseStore,
|
|
1138
|
+
* "uploads:",
|
|
1139
|
+
* jsonSerializer.serialize,
|
|
1140
|
+
* jsonSerializer.deserialize
|
|
1141
|
+
* );
|
|
1092
1142
|
*
|
|
1093
|
-
*
|
|
1094
|
-
*
|
|
1095
|
-
*
|
|
1096
|
-
*
|
|
1097
|
-
*
|
|
1098
|
-
* }
|
|
1143
|
+
* // Store and retrieve typed data
|
|
1144
|
+
* const program = Effect.gen(function* () {
|
|
1145
|
+
* const file: UploadFile = {
|
|
1146
|
+
* id: "file123",
|
|
1147
|
+
* offset: 0,
|
|
1148
|
+
* storage: { id: "s3", type: "s3" }
|
|
1149
|
+
* };
|
|
1150
|
+
*
|
|
1151
|
+
* // Automatic serialization
|
|
1152
|
+
* yield* uploadStore.set("file123", file);
|
|
1153
|
+
*
|
|
1154
|
+
* // Automatic deserialization with type safety
|
|
1155
|
+
* const retrieved = yield* uploadStore.get("file123");
|
|
1156
|
+
* console.log(retrieved.offset); // TypeScript knows this is a number
|
|
1157
|
+
* });
|
|
1099
1158
|
* ```
|
|
1100
1159
|
*/
|
|
1101
|
-
type
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
type: "paused";
|
|
1107
|
-
nodeId: string;
|
|
1108
|
-
executionState: {
|
|
1109
|
-
executionOrder: string[];
|
|
1110
|
-
currentIndex: number;
|
|
1111
|
-
inputs: Record<string, unknown>;
|
|
1112
|
-
};
|
|
1160
|
+
type KvStore<TData> = {
|
|
1161
|
+
readonly get: (key: string) => Effect.Effect<TData, UploadistaError>;
|
|
1162
|
+
readonly set: (key: string, value: TData) => Effect.Effect<void, UploadistaError>;
|
|
1163
|
+
readonly delete: (key: string) => Effect.Effect<void, UploadistaError>;
|
|
1164
|
+
readonly list?: () => Effect.Effect<Array<string>, UploadistaError>;
|
|
1113
1165
|
};
|
|
1114
1166
|
/**
|
|
1115
|
-
*
|
|
1116
|
-
*
|
|
1117
|
-
* Flows execute nodes in topological order, passing data between nodes through edges.
|
|
1118
|
-
* They support conditional execution, retry logic, pausable nodes, and event emission.
|
|
1119
|
-
*
|
|
1120
|
-
* @template TFlowInputSchema - Zod schema defining the shape of input data
|
|
1121
|
-
* @template TFlowOutputSchema - Zod schema defining the shape of output data
|
|
1122
|
-
* @template TRequirements - Effect requirements (services/contexts) needed by nodes
|
|
1167
|
+
* Typed wrapper class that adds serialization to a BaseKvStore.
|
|
1123
1168
|
*
|
|
1124
|
-
*
|
|
1125
|
-
*
|
|
1126
|
-
*
|
|
1127
|
-
* @property edges - Connections between nodes
|
|
1128
|
-
* @property inputSchema - Zod schema for validating flow inputs
|
|
1129
|
-
* @property outputSchema - Zod schema for validating flow outputs
|
|
1130
|
-
* @property onEvent - Optional callback for flow execution events
|
|
1131
|
-
* @property run - Executes the flow from the beginning
|
|
1132
|
-
* @property resume - Resumes a paused flow execution
|
|
1133
|
-
* @property validateTypes - Validates node type compatibility
|
|
1134
|
-
* @property validateInputs - Validates input data against schema
|
|
1135
|
-
* @property validateOutputs - Validates output data against schema
|
|
1169
|
+
* This class implements the KvStore interface by wrapping a BaseKvStore
|
|
1170
|
+
* and handling serialization/deserialization for a specific type. It also
|
|
1171
|
+
* adds a key prefix to isolate different data types in the same store.
|
|
1136
1172
|
*
|
|
1137
|
-
* @
|
|
1138
|
-
* Flows are created using {@link createFlowWithSchema}. The Effect-based design
|
|
1139
|
-
* allows for composable error handling, resource management, and dependency injection.
|
|
1173
|
+
* @template TData - The type of data to store
|
|
1140
1174
|
*
|
|
1141
1175
|
* @example
|
|
1142
1176
|
* ```typescript
|
|
1143
|
-
*
|
|
1144
|
-
*
|
|
1145
|
-
*
|
|
1146
|
-
*
|
|
1147
|
-
*
|
|
1148
|
-
*
|
|
1149
|
-
*
|
|
1150
|
-
* { source: "optimize", target: "storage" }
|
|
1151
|
-
* ],
|
|
1152
|
-
* inputSchema: z.object({ file: z.instanceof(File) }),
|
|
1153
|
-
* outputSchema: uploadFileSchema
|
|
1154
|
-
* });
|
|
1177
|
+
* // Create a typed store for UploadFile
|
|
1178
|
+
* const uploadFileStore = new TypedKvStore<UploadFile>(
|
|
1179
|
+
* baseKvStore,
|
|
1180
|
+
* "uploadista:upload-file:", // All keys will be prefixed
|
|
1181
|
+
* (data) => JSON.stringify(data),
|
|
1182
|
+
* (str) => JSON.parse(str) as UploadFile
|
|
1183
|
+
* );
|
|
1155
1184
|
*
|
|
1156
|
-
*
|
|
1157
|
-
*
|
|
1158
|
-
*
|
|
1159
|
-
*
|
|
1185
|
+
* // Use the store
|
|
1186
|
+
* const effect = Effect.gen(function* () {
|
|
1187
|
+
* const file: UploadFile = { ... };
|
|
1188
|
+
* yield* uploadFileStore.set("abc123", file);
|
|
1189
|
+
* // Internally stores at key "uploadista:upload-file:abc123"
|
|
1190
|
+
*
|
|
1191
|
+
* const retrieved = yield* uploadFileStore.get("abc123");
|
|
1192
|
+
* return retrieved;
|
|
1160
1193
|
* });
|
|
1194
|
+
*
|
|
1195
|
+
* // Custom serialization for binary data
|
|
1196
|
+
* const binaryStore = new TypedKvStore<Uint8Array>(
|
|
1197
|
+
* baseKvStore,
|
|
1198
|
+
* "binary:",
|
|
1199
|
+
* (data) => btoa(String.fromCharCode(...data)), // Base64 encode
|
|
1200
|
+
* (str) => Uint8Array.from(atob(str), c => c.charCodeAt(0)) // Base64 decode
|
|
1201
|
+
* );
|
|
1161
1202
|
* ```
|
|
1162
1203
|
*/
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
storageId: string;
|
|
1175
|
-
jobId: string;
|
|
1176
|
-
clientId: string | null;
|
|
1177
|
-
}) => Effect.Effect<FlowExecutionResult<Record<string, z.infer<TFlowOutputSchema>>>, UploadistaError, TRequirements>;
|
|
1178
|
-
resume: (args: {
|
|
1179
|
-
jobId: string;
|
|
1180
|
-
storageId: string;
|
|
1181
|
-
nodeResults: Record<string, unknown>;
|
|
1182
|
-
executionState: {
|
|
1183
|
-
executionOrder: string[];
|
|
1184
|
-
currentIndex: number;
|
|
1185
|
-
inputs: Record<string, z.infer<TFlowInputSchema>>;
|
|
1186
|
-
};
|
|
1187
|
-
clientId: string | null;
|
|
1188
|
-
}) => Effect.Effect<FlowExecutionResult<Record<string, z.infer<TFlowOutputSchema>>>, UploadistaError, TRequirements>;
|
|
1189
|
-
validateTypes: () => {
|
|
1190
|
-
isValid: boolean;
|
|
1191
|
-
errors: string[];
|
|
1192
|
-
};
|
|
1193
|
-
validateInputs: (inputs: unknown) => {
|
|
1194
|
-
isValid: boolean;
|
|
1195
|
-
errors: string[];
|
|
1196
|
-
};
|
|
1197
|
-
validateOutputs: (outputs: unknown) => {
|
|
1198
|
-
isValid: boolean;
|
|
1199
|
-
errors: string[];
|
|
1200
|
-
};
|
|
1201
|
-
};
|
|
1204
|
+
declare class TypedKvStore<TData> implements KvStore<TData> {
|
|
1205
|
+
private baseStore;
|
|
1206
|
+
private keyPrefix;
|
|
1207
|
+
private serialize;
|
|
1208
|
+
private deserialize;
|
|
1209
|
+
constructor(baseStore: BaseKvStore, keyPrefix: string, serialize: (data: TData) => string, deserialize: (str: string) => TData);
|
|
1210
|
+
get: (key: string) => Effect.Effect<TData, UploadistaError>;
|
|
1211
|
+
set: (key: string, value: TData) => Effect.Effect<void, UploadistaError>;
|
|
1212
|
+
delete: (key: string) => Effect.Effect<void, UploadistaError>;
|
|
1213
|
+
list: () => Effect.Effect<Array<string>, UploadistaError>;
|
|
1214
|
+
}
|
|
1202
1215
|
/**
|
|
1203
|
-
*
|
|
1216
|
+
* Default JSON serialization helpers.
|
|
1204
1217
|
*
|
|
1205
|
-
*
|
|
1206
|
-
*
|
|
1207
|
-
* handles errors with retries, and emits events during execution.
|
|
1218
|
+
* These functions provide standard JSON serialization for use with TypedKvStore.
|
|
1219
|
+
* They work with any JSON-serializable type.
|
|
1208
1220
|
*
|
|
1209
|
-
* @
|
|
1210
|
-
*
|
|
1211
|
-
*
|
|
1212
|
-
*
|
|
1213
|
-
*
|
|
1221
|
+
* @example
|
|
1222
|
+
* ```typescript
|
|
1223
|
+
* const store = new TypedKvStore<MyType>(
|
|
1224
|
+
* baseStore,
|
|
1225
|
+
* "mydata:",
|
|
1226
|
+
* jsonSerializer.serialize,
|
|
1227
|
+
* jsonSerializer.deserialize
|
|
1228
|
+
* );
|
|
1229
|
+
* ```
|
|
1230
|
+
*/
|
|
1231
|
+
declare const jsonSerializer: {
|
|
1232
|
+
serialize: <T>(data: T) => string;
|
|
1233
|
+
deserialize: <T>(str: string) => T;
|
|
1234
|
+
};
|
|
1235
|
+
declare const BaseKvStoreService_base: Context.TagClass<BaseKvStoreService, "BaseKvStore", BaseKvStore>;
|
|
1236
|
+
/**
|
|
1237
|
+
* Effect-TS context tag for the base untyped KV store.
|
|
1214
1238
|
*
|
|
1215
|
-
*
|
|
1216
|
-
*
|
|
1217
|
-
* @param config.name - Human-readable flow name
|
|
1218
|
-
* @param config.nodes - Array of nodes (can be plain nodes or Effects resolving to nodes)
|
|
1219
|
-
* @param config.edges - Array of edges connecting nodes
|
|
1220
|
-
* @param config.inputSchema - Zod schema for validating inputs
|
|
1221
|
-
* @param config.outputSchema - Zod schema for validating outputs
|
|
1222
|
-
* @param config.typeChecker - Optional custom type compatibility checker
|
|
1223
|
-
* @param config.onEvent - Optional event callback for monitoring execution
|
|
1239
|
+
* This is the low-level store that storage adapter implementations provide.
|
|
1240
|
+
* Most application code should use typed stores like UploadFileKVStore instead.
|
|
1224
1241
|
*
|
|
1225
|
-
* @
|
|
1242
|
+
* @example
|
|
1243
|
+
* ```typescript
|
|
1244
|
+
* // Provide a base store implementation
|
|
1245
|
+
* const baseStoreLayer = Layer.succeed(BaseKvStoreService, redisKvStore);
|
|
1226
1246
|
*
|
|
1227
|
-
*
|
|
1228
|
-
*
|
|
1229
|
-
*
|
|
1230
|
-
*
|
|
1247
|
+
* // Use in an Effect
|
|
1248
|
+
* const effect = Effect.gen(function* () {
|
|
1249
|
+
* const baseStore = yield* BaseKvStoreService;
|
|
1250
|
+
* yield* baseStore.set("raw-key", "raw-value");
|
|
1251
|
+
* });
|
|
1252
|
+
* ```
|
|
1253
|
+
*/
|
|
1254
|
+
declare class BaseKvStoreService extends BaseKvStoreService_base {}
|
|
1255
|
+
declare const UploadFileKVStore_base: Context.TagClass<UploadFileKVStore, "UploadFileKVStore", KvStore<UploadFile>>;
|
|
1256
|
+
/**
|
|
1257
|
+
* Effect-TS context tag for the UploadFile typed KV store.
|
|
1231
1258
|
*
|
|
1232
|
-
*
|
|
1233
|
-
*
|
|
1234
|
-
* - The flow performs topological sorting to determine execution order
|
|
1235
|
-
* - Conditional nodes are evaluated before execution
|
|
1236
|
-
* - Nodes can specify retry configuration with exponential backoff
|
|
1237
|
-
* - Pausable nodes can halt execution and resume later
|
|
1259
|
+
* This provides type-safe storage for UploadFile metadata. It's the primary
|
|
1260
|
+
* way to store and retrieve upload metadata in the system.
|
|
1238
1261
|
*
|
|
1239
1262
|
* @example
|
|
1240
1263
|
* ```typescript
|
|
1241
|
-
* const
|
|
1242
|
-
*
|
|
1243
|
-
*
|
|
1244
|
-
*
|
|
1245
|
-
*
|
|
1246
|
-
*
|
|
1247
|
-
*
|
|
1248
|
-
*
|
|
1249
|
-
*
|
|
1250
|
-
*
|
|
1251
|
-
*
|
|
1252
|
-
*
|
|
1253
|
-
*
|
|
1254
|
-
*
|
|
1255
|
-
* inputSchema: z.object({
|
|
1256
|
-
* file: z.instanceof(File),
|
|
1257
|
-
* metadata: z.record(z.string(), z.any()).optional()
|
|
1258
|
-
* }),
|
|
1259
|
-
* outputSchema: uploadFileSchema,
|
|
1260
|
-
* onEvent: (event) => Effect.gen(function* () {
|
|
1261
|
-
* console.log("Flow event:", event);
|
|
1262
|
-
* return { eventId: event.jobId };
|
|
1263
|
-
* })
|
|
1264
|
+
* const uploadEffect = Effect.gen(function* () {
|
|
1265
|
+
* const kvStore = yield* UploadFileKVStore;
|
|
1266
|
+
*
|
|
1267
|
+
* // Store upload metadata
|
|
1268
|
+
* const file: UploadFile = {
|
|
1269
|
+
* id: "upload123",
|
|
1270
|
+
* offset: 0,
|
|
1271
|
+
* storage: { id: "s3", type: "s3" }
|
|
1272
|
+
* };
|
|
1273
|
+
* yield* kvStore.set("upload123", file);
|
|
1274
|
+
*
|
|
1275
|
+
* // Retrieve with type safety
|
|
1276
|
+
* const retrieved = yield* kvStore.get("upload123");
|
|
1277
|
+
* return retrieved;
|
|
1264
1278
|
* });
|
|
1265
1279
|
* ```
|
|
1266
|
-
*
|
|
1267
|
-
* @see {@link Flow} for the returned flow type
|
|
1268
|
-
* @see {@link FlowConfig} for configuration options
|
|
1269
1280
|
*/
|
|
1270
|
-
declare
|
|
1271
|
-
//#endregion
|
|
1272
|
-
//#region src/flow/type-registry.d.ts
|
|
1281
|
+
declare class UploadFileKVStore extends UploadFileKVStore_base {}
|
|
1273
1282
|
/**
|
|
1274
|
-
*
|
|
1283
|
+
* Effect Layer that creates the UploadFileKVStore from a BaseKvStore.
|
|
1275
1284
|
*
|
|
1276
|
-
*
|
|
1277
|
-
*
|
|
1285
|
+
* This layer automatically wires up JSON serialization for UploadFile objects
|
|
1286
|
+
* with the "uploadista:upload-file:" key prefix.
|
|
1287
|
+
*
|
|
1288
|
+
* @example
|
|
1289
|
+
* ```typescript
|
|
1290
|
+
* const program = Effect.gen(function* () {
|
|
1291
|
+
* const kvStore = yield* UploadFileKVStore;
|
|
1292
|
+
* // Use the store...
|
|
1293
|
+
* }).pipe(
|
|
1294
|
+
* Effect.provide(uploadFileKvStore),
|
|
1295
|
+
* Effect.provide(baseStoreLayer)
|
|
1296
|
+
* );
|
|
1297
|
+
* ```
|
|
1278
1298
|
*/
|
|
1279
|
-
|
|
1299
|
+
declare const uploadFileKvStore: Layer.Layer<UploadFileKVStore, never, BaseKvStoreService>;
|
|
1300
|
+
declare const FlowJobKVStore_base: Context.TagClass<FlowJobKVStore, "FlowJobKVStore", KvStore<FlowJob>>;
|
|
1280
1301
|
/**
|
|
1281
|
-
*
|
|
1302
|
+
* Effect-TS context tag for the FlowJob typed KV store.
|
|
1282
1303
|
*
|
|
1283
|
-
*
|
|
1284
|
-
*
|
|
1285
|
-
* - A unique identifier with versioning
|
|
1286
|
-
* - A category (input or output)
|
|
1287
|
-
* - A Zod schema for runtime validation
|
|
1288
|
-
* - A semantic version for evolution
|
|
1289
|
-
* - A human-readable description
|
|
1304
|
+
* This provides type-safe storage for FlowJob metadata, tracking the
|
|
1305
|
+
* execution state of flow processing jobs.
|
|
1290
1306
|
*
|
|
1291
|
-
* @
|
|
1307
|
+
* @example
|
|
1308
|
+
* ```typescript
|
|
1309
|
+
* const flowEffect = Effect.gen(function* () {
|
|
1310
|
+
* const jobStore = yield* FlowJobKVStore;
|
|
1292
1311
|
*
|
|
1293
|
-
*
|
|
1294
|
-
*
|
|
1295
|
-
*
|
|
1296
|
-
*
|
|
1297
|
-
*
|
|
1312
|
+
* // Store job state
|
|
1313
|
+
* const job: FlowJob = {
|
|
1314
|
+
* id: "job123",
|
|
1315
|
+
* flowId: "flow_resize",
|
|
1316
|
+
* status: "running",
|
|
1317
|
+
* tasks: [],
|
|
1318
|
+
* createdAt: new Date(),
|
|
1319
|
+
* updatedAt: new Date()
|
|
1320
|
+
* };
|
|
1321
|
+
* yield* jobStore.set("job123", job);
|
|
1322
|
+
*
|
|
1323
|
+
* // Retrieve and check status
|
|
1324
|
+
* const retrieved = yield* jobStore.get("job123");
|
|
1325
|
+
* return retrieved.status;
|
|
1326
|
+
* });
|
|
1327
|
+
* ```
|
|
1328
|
+
*/
|
|
1329
|
+
declare class FlowJobKVStore extends FlowJobKVStore_base {}
|
|
1330
|
+
/**
|
|
1331
|
+
* Effect Layer that creates the FlowJobKVStore from a BaseKvStore.
|
|
1332
|
+
*
|
|
1333
|
+
* This layer automatically wires up JSON serialization for FlowJob objects
|
|
1334
|
+
* with the "uploadista:flow-job:" key prefix.
|
|
1298
1335
|
*
|
|
1299
1336
|
* @example
|
|
1300
1337
|
* ```typescript
|
|
1301
|
-
* const
|
|
1302
|
-
*
|
|
1303
|
-
*
|
|
1304
|
-
*
|
|
1305
|
-
*
|
|
1306
|
-
*
|
|
1307
|
-
*
|
|
1338
|
+
* const program = Effect.gen(function* () {
|
|
1339
|
+
* const jobStore = yield* FlowJobKVStore;
|
|
1340
|
+
* // Use the store...
|
|
1341
|
+
* }).pipe(
|
|
1342
|
+
* Effect.provide(flowJobKvStore),
|
|
1343
|
+
* Effect.provide(baseStoreLayer)
|
|
1344
|
+
* );
|
|
1308
1345
|
* ```
|
|
1309
1346
|
*/
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
schema: z.ZodSchema<TSchema>;
|
|
1314
|
-
version: string;
|
|
1315
|
-
description: string;
|
|
1316
|
-
}
|
|
1347
|
+
declare const flowJobKvStore: Layer.Layer<FlowJobKVStore, never, BaseKvStoreService>;
|
|
1348
|
+
//#endregion
|
|
1349
|
+
//#region src/types/data-store.d.ts
|
|
1317
1350
|
/**
|
|
1318
|
-
*
|
|
1351
|
+
* Options for writing data to a DataStore.
|
|
1319
1352
|
*
|
|
1320
|
-
* @
|
|
1353
|
+
* @property file_id - Unique identifier for the file being written
|
|
1354
|
+
* @property stream - Stream of byte chunks to write to storage
|
|
1355
|
+
* @property offset - Byte offset where writing should begin (for resumable uploads)
|
|
1321
1356
|
*/
|
|
1322
|
-
type
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
success: false;
|
|
1327
|
-
error: UploadistaError;
|
|
1357
|
+
type DataStoreWriteOptions = {
|
|
1358
|
+
file_id: string;
|
|
1359
|
+
stream: Stream.Stream<Uint8Array, UploadistaError>;
|
|
1360
|
+
offset: number;
|
|
1328
1361
|
};
|
|
1329
1362
|
/**
|
|
1330
|
-
*
|
|
1363
|
+
* Upload strategy type indicating how chunks are uploaded.
|
|
1331
1364
|
*
|
|
1332
|
-
*
|
|
1333
|
-
*
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
*
|
|
1365
|
+
* - `single`: Upload file in a single request (traditional upload)
|
|
1366
|
+
* - `parallel`: Upload file chunks in parallel (for large files)
|
|
1367
|
+
*/
|
|
1368
|
+
type UploadStrategy = "single" | "parallel";
|
|
1369
|
+
/**
|
|
1370
|
+
* Capabilities and constraints of a DataStore implementation.
|
|
1338
1371
|
*
|
|
1339
|
-
*
|
|
1340
|
-
*
|
|
1372
|
+
* This type describes what features a storage backend supports and what
|
|
1373
|
+
* limitations it has. Use this to determine the optimal upload strategy
|
|
1374
|
+
* and validate client requests.
|
|
1341
1375
|
*
|
|
1342
|
-
* @
|
|
1343
|
-
*
|
|
1344
|
-
* -
|
|
1345
|
-
* -
|
|
1346
|
-
*
|
|
1376
|
+
* @property supportsParallelUploads - Can upload chunks in parallel (e.g., S3 multipart)
|
|
1377
|
+
* @property supportsConcatenation - Can concatenate multiple uploads into one file
|
|
1378
|
+
* @property supportsDeferredLength - Can start upload without knowing final size
|
|
1379
|
+
* @property supportsResumableUploads - Can resume interrupted uploads from last offset
|
|
1380
|
+
* @property supportsTransactionalUploads - Guarantees atomic upload success/failure
|
|
1381
|
+
* @property maxConcurrentUploads - Maximum parallel upload parts (if parallel supported)
|
|
1382
|
+
* @property minChunkSize - Minimum size in bytes for each chunk (except last)
|
|
1383
|
+
* @property maxChunkSize - Maximum size in bytes for each chunk
|
|
1384
|
+
* @property maxParts - Maximum number of parts in a multipart upload
|
|
1385
|
+
* @property optimalChunkSize - Recommended chunk size for best performance
|
|
1386
|
+
* @property requiresOrderedChunks - Must receive chunks in sequential order
|
|
1387
|
+
* @property requiresMimeTypeValidation - Validates file MIME type matches declaration
|
|
1388
|
+
* @property maxValidationSize - Maximum file size for MIME type validation
|
|
1347
1389
|
*
|
|
1348
1390
|
* @example
|
|
1349
1391
|
* ```typescript
|
|
1350
|
-
*
|
|
1351
|
-
* flowTypeRegistry.register({
|
|
1352
|
-
* id: "webhook-output-v1",
|
|
1353
|
-
* category: "output",
|
|
1354
|
-
* schema: webhookResponseSchema,
|
|
1355
|
-
* version: "1.0.0",
|
|
1356
|
-
* description: "HTTP webhook notification output",
|
|
1357
|
-
* });
|
|
1392
|
+
* const capabilities = dataStore.getCapabilities();
|
|
1358
1393
|
*
|
|
1359
|
-
*
|
|
1360
|
-
*
|
|
1361
|
-
*
|
|
1362
|
-
*
|
|
1363
|
-
* }
|
|
1364
|
-
*
|
|
1365
|
-
*
|
|
1366
|
-
* const outputTypes = flowTypeRegistry.listByCategory("output");
|
|
1367
|
-
* console.log(outputTypes.map(t => t.id));
|
|
1368
|
-
*
|
|
1369
|
-
* // Validate data
|
|
1370
|
-
* const result = flowTypeRegistry.validate("webhook-output-v1", data);
|
|
1371
|
-
* if (result.success) {
|
|
1372
|
-
* // data is now typed according to the schema
|
|
1373
|
-
* processWebhookResponse(result.data);
|
|
1394
|
+
* if (capabilities.supportsParallelUploads && fileSize > 10_000_000) {
|
|
1395
|
+
* // Use parallel upload for large files
|
|
1396
|
+
* const chunkSize = capabilities.optimalChunkSize || 5_242_880; // 5MB default
|
|
1397
|
+
* uploadInParallel(file, chunkSize);
|
|
1398
|
+
* } else {
|
|
1399
|
+
* // Use single upload
|
|
1400
|
+
* uploadAsSingleChunk(file);
|
|
1374
1401
|
* }
|
|
1375
1402
|
* ```
|
|
1376
1403
|
*/
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
* import { z } from "zod";
|
|
1393
|
-
*
|
|
1394
|
-
* flowTypeRegistry.register({
|
|
1395
|
-
* id: "description-output-v1",
|
|
1396
|
-
* category: "output",
|
|
1397
|
-
* schema: z.object({
|
|
1398
|
-
* description: z.string(),
|
|
1399
|
-
* confidence: z.number().min(0).max(1),
|
|
1400
|
-
* tags: z.array(z.string()).optional(),
|
|
1401
|
-
* }),
|
|
1402
|
-
* version: "1.0.0",
|
|
1403
|
-
* description: "AI-generated image description with confidence score",
|
|
1404
|
-
* });
|
|
1405
|
-
* ```
|
|
1406
|
-
*/
|
|
1407
|
-
register<T>(definition: NodeTypeDefinition<T>): void;
|
|
1408
|
-
/**
|
|
1409
|
-
* Retrieve a registered type definition by its ID.
|
|
1410
|
-
*
|
|
1411
|
-
* @param id - The unique type identifier (e.g., "storage-output-v1")
|
|
1412
|
-
* @returns The type definition if found, undefined otherwise
|
|
1413
|
-
*
|
|
1414
|
-
* @example
|
|
1415
|
-
* ```typescript
|
|
1416
|
-
* const def = flowTypeRegistry.get("storage-output-v1");
|
|
1417
|
-
* if (def) {
|
|
1418
|
-
* console.log(`Found ${def.description} (v${def.version})`);
|
|
1419
|
-
* } else {
|
|
1420
|
-
* console.warn("Type not registered");
|
|
1421
|
-
* }
|
|
1422
|
-
* ```
|
|
1423
|
-
*/
|
|
1424
|
-
get(id: string): NodeTypeDefinition<unknown> | undefined;
|
|
1425
|
-
/**
|
|
1426
|
-
* List all registered types in a specific category.
|
|
1427
|
-
*
|
|
1428
|
-
* @param category - The node category to filter by ("input" or "output")
|
|
1429
|
-
* @returns Array of type definitions in the specified category
|
|
1430
|
-
*
|
|
1431
|
-
* @example
|
|
1432
|
-
* ```typescript
|
|
1433
|
-
* // List all registered output types
|
|
1434
|
-
* const outputTypes = flowTypeRegistry.listByCategory("output");
|
|
1435
|
-
* console.log("Available output types:");
|
|
1436
|
-
* for (const type of outputTypes) {
|
|
1437
|
-
* console.log(`- ${type.id}: ${type.description}`);
|
|
1438
|
-
* }
|
|
1439
|
-
* ```
|
|
1440
|
-
*/
|
|
1441
|
-
listByCategory(category: NodeTypeCategory): NodeTypeDefinition<unknown>[];
|
|
1442
|
-
/**
|
|
1443
|
-
* Validate data against a registered type's schema.
|
|
1444
|
-
*
|
|
1445
|
-
* This method performs runtime validation using the Zod schema associated with
|
|
1446
|
-
* the type. If validation succeeds, the data is returned with proper typing.
|
|
1447
|
-
* If validation fails, an UploadistaError is returned with details.
|
|
1448
|
-
*
|
|
1449
|
-
* @template T - The expected TypeScript type after validation
|
|
1450
|
-
* @param typeId - The ID of the registered type to validate against
|
|
1451
|
-
* @param data - The data to validate
|
|
1452
|
-
* @returns A result object with either the validated data or an error
|
|
1453
|
-
*
|
|
1454
|
-
* @example
|
|
1455
|
-
* ```typescript
|
|
1456
|
-
* const result = flowTypeRegistry.validate("storage-output-v1", unknownData);
|
|
1457
|
-
*
|
|
1458
|
-
* if (result.success) {
|
|
1459
|
-
* // TypeScript knows result.data is an UploadFile
|
|
1460
|
-
* console.log(`File stored at: ${result.data.url}`);
|
|
1461
|
-
* } else {
|
|
1462
|
-
* console.error(`Validation failed: ${result.error.body}`);
|
|
1463
|
-
* }
|
|
1464
|
-
* ```
|
|
1465
|
-
*/
|
|
1466
|
-
validate<T>(typeId: string, data: unknown): ValidationResult<T>;
|
|
1467
|
-
/**
|
|
1468
|
-
* Check if a type is registered.
|
|
1469
|
-
*
|
|
1470
|
-
* @param id - The unique type identifier to check
|
|
1471
|
-
* @returns True if the type is registered, false otherwise
|
|
1472
|
-
*
|
|
1473
|
-
* @example
|
|
1474
|
-
* ```typescript
|
|
1475
|
-
* if (flowTypeRegistry.has("custom-output-v1")) {
|
|
1476
|
-
* console.log("Custom output type is available");
|
|
1477
|
-
* }
|
|
1478
|
-
* ```
|
|
1479
|
-
*/
|
|
1480
|
-
has(id: string): boolean;
|
|
1481
|
-
/**
|
|
1482
|
-
* Get the total number of registered types.
|
|
1483
|
-
*
|
|
1484
|
-
* @returns The count of registered types
|
|
1485
|
-
*
|
|
1486
|
-
* @example
|
|
1487
|
-
* ```typescript
|
|
1488
|
-
* console.log(`Registry contains ${flowTypeRegistry.size()} types`);
|
|
1489
|
-
* ```
|
|
1490
|
-
*/
|
|
1491
|
-
size(): number;
|
|
1492
|
-
}
|
|
1404
|
+
type DataStoreCapabilities = {
|
|
1405
|
+
supportsParallelUploads: boolean;
|
|
1406
|
+
supportsConcatenation: boolean;
|
|
1407
|
+
supportsDeferredLength: boolean;
|
|
1408
|
+
supportsResumableUploads: boolean;
|
|
1409
|
+
supportsTransactionalUploads: boolean;
|
|
1410
|
+
maxConcurrentUploads?: number;
|
|
1411
|
+
minChunkSize?: number;
|
|
1412
|
+
maxChunkSize?: number;
|
|
1413
|
+
maxParts?: number;
|
|
1414
|
+
optimalChunkSize?: number;
|
|
1415
|
+
requiresOrderedChunks: boolean;
|
|
1416
|
+
requiresMimeTypeValidation?: boolean;
|
|
1417
|
+
maxValidationSize?: number;
|
|
1418
|
+
};
|
|
1493
1419
|
/**
|
|
1494
|
-
*
|
|
1420
|
+
* Core interface for all storage backend implementations.
|
|
1495
1421
|
*
|
|
1496
|
-
*
|
|
1497
|
-
*
|
|
1422
|
+
* DataStore abstracts file storage operations across different backends
|
|
1423
|
+
* (S3, Azure Blob, GCS, local filesystem, etc.). All storage adapters
|
|
1424
|
+
* must implement this interface.
|
|
1425
|
+
*
|
|
1426
|
+
* @template TData - The data type stored (typically UploadFile)
|
|
1427
|
+
*
|
|
1428
|
+
* @property bucket - Optional storage bucket or container name
|
|
1429
|
+
* @property path - Optional base path prefix for all stored files
|
|
1430
|
+
* @property create - Creates a new file record in storage
|
|
1431
|
+
* @property remove - Deletes a file from storage
|
|
1432
|
+
* @property read - Reads complete file contents as bytes
|
|
1433
|
+
* @property write - Writes data stream to storage at specified offset
|
|
1434
|
+
* @property deleteExpired - Optional cleanup of expired files
|
|
1435
|
+
* @property getCapabilities - Returns storage backend capabilities
|
|
1436
|
+
* @property validateUploadStrategy - Validates if strategy is supported
|
|
1498
1437
|
*
|
|
1499
1438
|
* @example
|
|
1500
1439
|
* ```typescript
|
|
1501
|
-
*
|
|
1502
|
-
*
|
|
1503
|
-
*
|
|
1504
|
-
*
|
|
1505
|
-
* id: "my-output-v1",
|
|
1506
|
-
* category: "output",
|
|
1507
|
-
* schema: mySchema,
|
|
1508
|
-
* version: "1.0.0",
|
|
1509
|
-
* description: "My custom output type",
|
|
1510
|
-
* });
|
|
1440
|
+
* // Implement a custom DataStore
|
|
1441
|
+
* const myDataStore: DataStore<UploadFile> = {
|
|
1442
|
+
* bucket: "my-uploads",
|
|
1443
|
+
* path: "files/",
|
|
1511
1444
|
*
|
|
1512
|
-
*
|
|
1513
|
-
*
|
|
1514
|
-
*
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
//#endregion
|
|
1518
|
-
//#region src/flow/node-types/index.d.ts
|
|
1519
|
-
/**
|
|
1520
|
-
* Built-in node type registrations for the flow engine.
|
|
1445
|
+
* create: (file) => Effect.gen(function* () {
|
|
1446
|
+
* // Store file metadata
|
|
1447
|
+
* yield* saveMetadata(file);
|
|
1448
|
+
* return file;
|
|
1449
|
+
* }),
|
|
1521
1450
|
*
|
|
1522
|
-
*
|
|
1523
|
-
*
|
|
1451
|
+
* write: ({ file_id, stream, offset }, { onProgress }) => Effect.gen(function* () {
|
|
1452
|
+
* // Write chunks to storage
|
|
1453
|
+
* let bytesWritten = offset;
|
|
1454
|
+
* yield* Stream.runForEach(stream, (chunk) => Effect.sync(() => {
|
|
1455
|
+
* writeChunk(file_id, chunk, bytesWritten);
|
|
1456
|
+
* bytesWritten += chunk.byteLength;
|
|
1457
|
+
* onProgress?.(chunk.byteLength);
|
|
1458
|
+
* }));
|
|
1459
|
+
* return bytesWritten;
|
|
1460
|
+
* }),
|
|
1524
1461
|
*
|
|
1525
|
-
*
|
|
1462
|
+
* read: (file_id) => Effect.gen(function* () {
|
|
1463
|
+
* // Read complete file
|
|
1464
|
+
* const data = yield* readFromStorage(file_id);
|
|
1465
|
+
* return data;
|
|
1466
|
+
* }),
|
|
1526
1467
|
*
|
|
1527
|
-
*
|
|
1528
|
-
*
|
|
1529
|
-
*
|
|
1468
|
+
* remove: (file_id) => Effect.gen(function* () {
|
|
1469
|
+
* yield* deleteFromStorage(file_id);
|
|
1470
|
+
* }),
|
|
1530
1471
|
*
|
|
1531
|
-
*
|
|
1532
|
-
*
|
|
1533
|
-
*
|
|
1534
|
-
*
|
|
1535
|
-
*
|
|
1472
|
+
* getCapabilities: () => ({
|
|
1473
|
+
* supportsParallelUploads: true,
|
|
1474
|
+
* supportsConcatenation: false,
|
|
1475
|
+
* supportsDeferredLength: true,
|
|
1476
|
+
* supportsResumableUploads: true,
|
|
1477
|
+
* supportsTransactionalUploads: false,
|
|
1478
|
+
* maxConcurrentUploads: 10,
|
|
1479
|
+
* optimalChunkSize: 5_242_880, // 5MB
|
|
1480
|
+
* requiresOrderedChunks: false,
|
|
1481
|
+
* }),
|
|
1536
1482
|
*
|
|
1537
|
-
*
|
|
1538
|
-
*
|
|
1539
|
-
*
|
|
1483
|
+
* validateUploadStrategy: (strategy) =>
|
|
1484
|
+
* Effect.succeed(strategy === "parallel" || strategy === "single"),
|
|
1485
|
+
* };
|
|
1540
1486
|
* ```
|
|
1541
1487
|
*/
|
|
1488
|
+
type DataStore<TData = unknown> = {
|
|
1489
|
+
readonly bucket?: string;
|
|
1490
|
+
readonly path?: string;
|
|
1491
|
+
readonly create: (file: TData) => Effect.Effect<TData, UploadistaError>;
|
|
1492
|
+
readonly remove: (file_id: string) => Effect.Effect<void, UploadistaError>;
|
|
1493
|
+
readonly read: (file_id: string) => Effect.Effect<Uint8Array, UploadistaError>;
|
|
1494
|
+
readonly write: (options: DataStoreWriteOptions, dependencies: {
|
|
1495
|
+
onProgress?: (chunkSize: number) => void;
|
|
1496
|
+
}) => Effect.Effect<number, UploadistaError>;
|
|
1497
|
+
readonly deleteExpired?: () => Effect.Effect<number, UploadistaError>;
|
|
1498
|
+
readonly getCapabilities: () => DataStoreCapabilities;
|
|
1499
|
+
readonly validateUploadStrategy: (strategy: UploadStrategy) => Effect.Effect<boolean, never>;
|
|
1500
|
+
};
|
|
1501
|
+
declare const UploadFileDataStore_base: Context.TagClass<UploadFileDataStore, "UploadFileDataStore", DataStore<UploadFile>>;
|
|
1542
1502
|
/**
|
|
1543
|
-
*
|
|
1503
|
+
* Effect-TS context tag for UploadFile DataStore.
|
|
1544
1504
|
*
|
|
1545
|
-
* Use
|
|
1546
|
-
*
|
|
1505
|
+
* Use this tag to access the primary DataStore in an Effect context.
|
|
1506
|
+
* This is the standard storage backend for uploaded files.
|
|
1547
1507
|
*
|
|
1548
1508
|
* @example
|
|
1549
1509
|
* ```typescript
|
|
1550
|
-
*
|
|
1551
|
-
*
|
|
1552
|
-
*
|
|
1553
|
-
*
|
|
1554
|
-
*
|
|
1555
|
-
* });
|
|
1510
|
+
* const uploadEffect = Effect.gen(function* () {
|
|
1511
|
+
* const dataStore = yield* UploadFileDataStore;
|
|
1512
|
+
* const file = yield* dataStore.create(uploadFile);
|
|
1513
|
+
* return file;
|
|
1514
|
+
* });
|
|
1556
1515
|
* ```
|
|
1557
1516
|
*/
|
|
1558
|
-
declare
|
|
1559
|
-
declare const
|
|
1560
|
-
//#endregion
|
|
1561
|
-
//#region src/types/kv-store.d.ts
|
|
1517
|
+
declare class UploadFileDataStore extends UploadFileDataStore_base {}
|
|
1518
|
+
declare const BufferedUploadFileDataStore_base: Context.TagClass<BufferedUploadFileDataStore, "BufferedUploadFileDataStore", DataStore<UploadFile>>;
|
|
1562
1519
|
/**
|
|
1563
|
-
*
|
|
1564
|
-
*
|
|
1565
|
-
* This is the low-level interface that storage adapters implement.
|
|
1566
|
-
* It stores raw string values without type safety or serialization.
|
|
1520
|
+
* Effect-TS context tag for buffered/temporary DataStore.
|
|
1567
1521
|
*
|
|
1568
|
-
*
|
|
1569
|
-
*
|
|
1570
|
-
* @property delete - Removes a value by key
|
|
1571
|
-
* @property list - Optional operation to list all keys with a given prefix
|
|
1522
|
+
* This is an optional storage backend used for temporary or intermediate files
|
|
1523
|
+
* during flow processing. Not all implementations provide a buffered store.
|
|
1572
1524
|
*
|
|
1573
1525
|
* @example
|
|
1574
1526
|
* ```typescript
|
|
1575
|
-
*
|
|
1576
|
-
*
|
|
1577
|
-
*
|
|
1578
|
-
*
|
|
1579
|
-
*
|
|
1580
|
-
*
|
|
1581
|
-
*
|
|
1582
|
-
* set: (key, value) => Effect.tryPromise({
|
|
1583
|
-
* try: () => redis.set(key, value),
|
|
1584
|
-
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1585
|
-
* }),
|
|
1586
|
-
*
|
|
1587
|
-
* delete: (key) => Effect.tryPromise({
|
|
1588
|
-
* try: () => redis.del(key),
|
|
1589
|
-
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1590
|
-
* }),
|
|
1591
|
-
*
|
|
1592
|
-
* list: (prefix) => Effect.tryPromise({
|
|
1593
|
-
* try: () => redis.keys(`${prefix}*`),
|
|
1594
|
-
* catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
|
|
1595
|
-
* })
|
|
1596
|
-
* };
|
|
1527
|
+
* const processEffect = Effect.gen(function* () {
|
|
1528
|
+
* const bufferedStore = yield* BufferedUploadFileDataStore;
|
|
1529
|
+
* // Store intermediate processing results
|
|
1530
|
+
* const tempFile = yield* bufferedStore.create(intermediateFile);
|
|
1531
|
+
* return tempFile;
|
|
1532
|
+
* });
|
|
1597
1533
|
* ```
|
|
1598
1534
|
*/
|
|
1599
|
-
|
|
1600
|
-
readonly get: (key: string) => Effect.Effect<string | null, UploadistaError>;
|
|
1601
|
-
readonly set: (key: string, value: string) => Effect.Effect<void, UploadistaError>;
|
|
1602
|
-
readonly delete: (key: string) => Effect.Effect<void, UploadistaError>;
|
|
1603
|
-
readonly list?: (keyPrefix: string) => Effect.Effect<Array<string>, UploadistaError>;
|
|
1604
|
-
}
|
|
1535
|
+
declare class BufferedUploadFileDataStore extends BufferedUploadFileDataStore_base {}
|
|
1605
1536
|
/**
|
|
1606
|
-
*
|
|
1537
|
+
* Service interface for managing multiple DataStore instances.
|
|
1607
1538
|
*
|
|
1608
|
-
* This
|
|
1609
|
-
*
|
|
1610
|
-
* for manual JSON.stringify/parse calls.
|
|
1539
|
+
* This allows routing files to different storage backends based on
|
|
1540
|
+
* storageId (e.g., different S3 buckets, Azure containers, or storage tiers).
|
|
1611
1541
|
*
|
|
1612
|
-
* @
|
|
1542
|
+
* @property getDataStore - Retrieves the appropriate DataStore for a given storage ID
|
|
1543
|
+
* @property bufferedDataStore - Optional temporary storage for intermediate files
|
|
1544
|
+
*/
|
|
1545
|
+
type UploadFileDataStoresShape = {
|
|
1546
|
+
getDataStore: (storageId: string, clientId: string | null) => Effect.Effect<DataStore<UploadFile>, UploadistaError>;
|
|
1547
|
+
bufferedDataStore: Effect.Effect<DataStore<UploadFile> | undefined, UploadistaError>;
|
|
1548
|
+
};
|
|
1549
|
+
declare const UploadFileDataStores_base: Context.TagClass<UploadFileDataStores, "UploadFileDataStores", UploadFileDataStoresShape>;
|
|
1550
|
+
/**
|
|
1551
|
+
* Effect-TS context tag for the DataStore routing service.
|
|
1613
1552
|
*
|
|
1614
|
-
*
|
|
1615
|
-
* @property set - Serializes and stores a value
|
|
1616
|
-
* @property delete - Removes a value by key
|
|
1617
|
-
* @property list - Optional operation to list all keys (without prefix)
|
|
1553
|
+
* Provides access to multiple DataStore instances with routing logic.
|
|
1618
1554
|
*
|
|
1619
1555
|
* @example
|
|
1620
1556
|
* ```typescript
|
|
1621
|
-
*
|
|
1622
|
-
*
|
|
1623
|
-
*
|
|
1624
|
-
* "
|
|
1625
|
-
*
|
|
1626
|
-
*
|
|
1627
|
-
* );
|
|
1628
|
-
*
|
|
1629
|
-
* // Store and retrieve typed data
|
|
1630
|
-
* const program = Effect.gen(function* () {
|
|
1631
|
-
* const file: UploadFile = {
|
|
1632
|
-
* id: "file123",
|
|
1633
|
-
* offset: 0,
|
|
1634
|
-
* storage: { id: "s3", type: "s3" }
|
|
1635
|
-
* };
|
|
1636
|
-
*
|
|
1637
|
-
* // Automatic serialization
|
|
1638
|
-
* yield* uploadStore.set("file123", file);
|
|
1639
|
-
*
|
|
1640
|
-
* // Automatic deserialization with type safety
|
|
1641
|
-
* const retrieved = yield* uploadStore.get("file123");
|
|
1642
|
-
* console.log(retrieved.offset); // TypeScript knows this is a number
|
|
1557
|
+
* const uploadEffect = Effect.gen(function* () {
|
|
1558
|
+
* const dataStores = yield* UploadFileDataStores;
|
|
1559
|
+
* // Route to specific storage based on storageId
|
|
1560
|
+
* const dataStore = yield* dataStores.getDataStore("s3-production", clientId);
|
|
1561
|
+
* const file = yield* dataStore.create(uploadFile);
|
|
1562
|
+
* return file;
|
|
1643
1563
|
* });
|
|
1644
1564
|
* ```
|
|
1645
1565
|
*/
|
|
1646
|
-
|
|
1647
|
-
readonly get: (key: string) => Effect.Effect<TData, UploadistaError>;
|
|
1648
|
-
readonly set: (key: string, value: TData) => Effect.Effect<void, UploadistaError>;
|
|
1649
|
-
readonly delete: (key: string) => Effect.Effect<void, UploadistaError>;
|
|
1650
|
-
readonly list?: () => Effect.Effect<Array<string>, UploadistaError>;
|
|
1651
|
-
};
|
|
1566
|
+
declare class UploadFileDataStores extends UploadFileDataStores_base {}
|
|
1652
1567
|
/**
|
|
1653
|
-
*
|
|
1654
|
-
*
|
|
1655
|
-
* This class implements the KvStore interface by wrapping a BaseKvStore
|
|
1656
|
-
* and handling serialization/deserialization for a specific type. It also
|
|
1657
|
-
* adds a key prefix to isolate different data types in the same store.
|
|
1568
|
+
* Simplified DataStore configuration for easy setup.
|
|
1658
1569
|
*
|
|
1659
|
-
*
|
|
1570
|
+
* This type allows flexible configuration:
|
|
1571
|
+
* - Single DataStore instance
|
|
1572
|
+
* - Multiple named stores with routing
|
|
1573
|
+
* - Effect that resolves to a DataStore
|
|
1574
|
+
* - Pre-built Effect Layer
|
|
1660
1575
|
*
|
|
1661
1576
|
* @example
|
|
1662
1577
|
* ```typescript
|
|
1663
|
-
* //
|
|
1664
|
-
* const
|
|
1665
|
-
* baseKvStore,
|
|
1666
|
-
* "uploadista:upload-file:", // All keys will be prefixed
|
|
1667
|
-
* (data) => JSON.stringify(data),
|
|
1668
|
-
* (str) => JSON.parse(str) as UploadFile
|
|
1669
|
-
* );
|
|
1578
|
+
* // Single store
|
|
1579
|
+
* const config: DataStoreConfig = s3DataStore;
|
|
1670
1580
|
*
|
|
1671
|
-
* //
|
|
1672
|
-
* const
|
|
1673
|
-
*
|
|
1674
|
-
*
|
|
1675
|
-
*
|
|
1581
|
+
* // Multiple stores with routing
|
|
1582
|
+
* const config: DataStoreConfig = {
|
|
1583
|
+
* stores: {
|
|
1584
|
+
* "s3-prod": s3ProdStore,
|
|
1585
|
+
* "s3-dev": s3DevStore,
|
|
1586
|
+
* "local": localFileStore,
|
|
1587
|
+
* },
|
|
1588
|
+
* default: "s3-prod"
|
|
1589
|
+
* };
|
|
1676
1590
|
*
|
|
1677
|
-
*
|
|
1678
|
-
*
|
|
1591
|
+
* // Effect that creates a store
|
|
1592
|
+
* const config: DataStoreConfig = Effect.gen(function* () {
|
|
1593
|
+
* const kvStore = yield* UploadFileKVStore;
|
|
1594
|
+
* return s3Store(kvStore);
|
|
1679
1595
|
* });
|
|
1680
1596
|
*
|
|
1681
|
-
* //
|
|
1682
|
-
* const
|
|
1683
|
-
* baseKvStore,
|
|
1684
|
-
* "binary:",
|
|
1685
|
-
* (data) => btoa(String.fromCharCode(...data)), // Base64 encode
|
|
1686
|
-
* (str) => Uint8Array.from(atob(str), c => c.charCodeAt(0)) // Base64 decode
|
|
1687
|
-
* );
|
|
1597
|
+
* // Pre-built Layer
|
|
1598
|
+
* const config: DataStoreConfig = Layer.succeed(UploadFileDataStores, {...});
|
|
1688
1599
|
* ```
|
|
1689
1600
|
*/
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
private deserialize;
|
|
1695
|
-
constructor(baseStore: BaseKvStore, keyPrefix: string, serialize: (data: TData) => string, deserialize: (str: string) => TData);
|
|
1696
|
-
get: (key: string) => Effect.Effect<TData, UploadistaError>;
|
|
1697
|
-
set: (key: string, value: TData) => Effect.Effect<void, UploadistaError>;
|
|
1698
|
-
delete: (key: string) => Effect.Effect<void, UploadistaError>;
|
|
1699
|
-
list: () => Effect.Effect<Array<string>, UploadistaError>;
|
|
1700
|
-
}
|
|
1601
|
+
type DataStoreConfig = DataStore<UploadFile> | Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore> | {
|
|
1602
|
+
stores: Record<string, DataStore<UploadFile> | Effect.Effect<DataStore<UploadFile>, never, UploadFileKVStore>>;
|
|
1603
|
+
default?: string;
|
|
1604
|
+
} | Layer.Layer<UploadFileDataStores, never, UploadFileKVStore>;
|
|
1701
1605
|
/**
|
|
1702
|
-
*
|
|
1606
|
+
* Type guard to check if a value is a DataStore instance.
|
|
1703
1607
|
*
|
|
1704
|
-
*
|
|
1705
|
-
*
|
|
1608
|
+
* @param config - The value to check
|
|
1609
|
+
* @returns True if the value is a DataStore
|
|
1706
1610
|
*
|
|
1707
1611
|
* @example
|
|
1708
1612
|
* ```typescript
|
|
1709
|
-
*
|
|
1710
|
-
*
|
|
1711
|
-
*
|
|
1712
|
-
* jsonSerializer.serialize,
|
|
1713
|
-
* jsonSerializer.deserialize
|
|
1714
|
-
* );
|
|
1613
|
+
* if (isDataStore(config)) {
|
|
1614
|
+
* const capabilities = config.getCapabilities();
|
|
1615
|
+
* }
|
|
1715
1616
|
* ```
|
|
1716
1617
|
*/
|
|
1717
|
-
declare const
|
|
1718
|
-
serialize: <T>(data: T) => string;
|
|
1719
|
-
deserialize: <T>(str: string) => T;
|
|
1720
|
-
};
|
|
1721
|
-
declare const BaseKvStoreService_base: Context.TagClass<BaseKvStoreService, "BaseKvStore", BaseKvStore>;
|
|
1618
|
+
declare const isDataStore: (config: DataStoreConfig) => config is DataStore<UploadFile>;
|
|
1722
1619
|
/**
|
|
1723
|
-
*
|
|
1620
|
+
* Creates an Effect Layer from simplified DataStoreConfig.
|
|
1724
1621
|
*
|
|
1725
|
-
* This
|
|
1726
|
-
*
|
|
1622
|
+
* This function converts any DataStoreConfig format into a proper Effect Layer
|
|
1623
|
+
* that can be provided to the UploadFileDataStores context tag.
|
|
1624
|
+
*
|
|
1625
|
+
* It handles:
|
|
1626
|
+
* - Single DataStore: Wraps in a Layer that always returns that store
|
|
1627
|
+
* - Multiple stores: Creates routing logic with optional default
|
|
1628
|
+
* - Effect<DataStore>: Executes the Effect and wraps the result
|
|
1629
|
+
* - Layer: Returns as-is
|
|
1630
|
+
*
|
|
1631
|
+
* @param config - The DataStore configuration
|
|
1632
|
+
* @returns A Layer that provides UploadFileDataStores service
|
|
1727
1633
|
*
|
|
1728
1634
|
* @example
|
|
1729
1635
|
* ```typescript
|
|
1730
|
-
* //
|
|
1731
|
-
* const
|
|
1636
|
+
* // Create from single store
|
|
1637
|
+
* const layer = await createDataStoreLayer(s3DataStore);
|
|
1732
1638
|
*
|
|
1733
|
-
* //
|
|
1734
|
-
* const
|
|
1735
|
-
*
|
|
1736
|
-
*
|
|
1639
|
+
* // Create from multiple stores
|
|
1640
|
+
* const layer = await createDataStoreLayer({
|
|
1641
|
+
* stores: {
|
|
1642
|
+
* "production": s3Store,
|
|
1643
|
+
* "development": localStore,
|
|
1644
|
+
* },
|
|
1645
|
+
* default: "development"
|
|
1737
1646
|
* });
|
|
1647
|
+
*
|
|
1648
|
+
* // Use the layer
|
|
1649
|
+
* const program = Effect.gen(function* () {
|
|
1650
|
+
* const stores = yield* UploadFileDataStores;
|
|
1651
|
+
* const store = yield* stores.getDataStore("production", null);
|
|
1652
|
+
* return store;
|
|
1653
|
+
* }).pipe(Effect.provide(layer));
|
|
1738
1654
|
* ```
|
|
1739
1655
|
*/
|
|
1740
|
-
declare
|
|
1741
|
-
|
|
1656
|
+
declare const createDataStoreLayer: (config: DataStoreConfig) => Promise<Layer.Layer<UploadFileDataStores, never, UploadFileKVStore>>;
|
|
1657
|
+
//#endregion
|
|
1658
|
+
//#region src/flow/flow.d.ts
|
|
1742
1659
|
/**
|
|
1743
|
-
*
|
|
1744
|
-
*
|
|
1745
|
-
* This provides type-safe storage for UploadFile metadata. It's the primary
|
|
1746
|
-
* way to store and retrieve upload metadata in the system.
|
|
1747
|
-
*
|
|
1748
|
-
* @example
|
|
1749
|
-
* ```typescript
|
|
1750
|
-
* const uploadEffect = Effect.gen(function* () {
|
|
1751
|
-
* const kvStore = yield* UploadFileKVStore;
|
|
1660
|
+
* Serialized flow data for storage and transport.
|
|
1661
|
+
* Contains the minimal information needed to reconstruct a flow.
|
|
1752
1662
|
*
|
|
1753
|
-
*
|
|
1754
|
-
*
|
|
1755
|
-
*
|
|
1756
|
-
*
|
|
1757
|
-
* storage: { id: "s3", type: "s3" }
|
|
1758
|
-
* };
|
|
1759
|
-
* yield* kvStore.set("upload123", file);
|
|
1760
|
-
*
|
|
1761
|
-
* // Retrieve with type safety
|
|
1762
|
-
* const retrieved = yield* kvStore.get("upload123");
|
|
1763
|
-
* return retrieved;
|
|
1764
|
-
* });
|
|
1765
|
-
* ```
|
|
1663
|
+
* @property id - Unique flow identifier
|
|
1664
|
+
* @property name - Human-readable flow name
|
|
1665
|
+
* @property nodes - Array of node data (without execution logic)
|
|
1666
|
+
* @property edges - Connections between nodes defining data flow
|
|
1766
1667
|
*/
|
|
1767
|
-
|
|
1668
|
+
type FlowData = {
|
|
1669
|
+
id: string;
|
|
1670
|
+
name: string;
|
|
1671
|
+
nodes: FlowNodeData[];
|
|
1672
|
+
edges: FlowEdge[];
|
|
1673
|
+
};
|
|
1768
1674
|
/**
|
|
1769
|
-
*
|
|
1675
|
+
* Extracts serializable flow data from a Flow instance.
|
|
1676
|
+
* Useful for storing flow definitions or sending them over the network.
|
|
1770
1677
|
*
|
|
1771
|
-
*
|
|
1772
|
-
*
|
|
1678
|
+
* @template TRequirements - Effect requirements for the flow
|
|
1679
|
+
* @param flow - Flow instance to extract data from
|
|
1680
|
+
* @returns Serializable flow data without execution logic
|
|
1773
1681
|
*
|
|
1774
1682
|
* @example
|
|
1775
1683
|
* ```typescript
|
|
1776
|
-
* const
|
|
1777
|
-
*
|
|
1778
|
-
*
|
|
1779
|
-
* }).pipe(
|
|
1780
|
-
* Effect.provide(uploadFileKvStore),
|
|
1781
|
-
* Effect.provide(baseStoreLayer)
|
|
1782
|
-
* );
|
|
1684
|
+
* const flowData = getFlowData(myFlow);
|
|
1685
|
+
* // Store in database or send to client
|
|
1686
|
+
* await db.flows.save(flowData);
|
|
1783
1687
|
* ```
|
|
1784
1688
|
*/
|
|
1785
|
-
declare const
|
|
1786
|
-
declare const FlowJobKVStore_base: Context.TagClass<FlowJobKVStore, "FlowJobKVStore", KvStore<FlowJob>>;
|
|
1689
|
+
declare const getFlowData: <TRequirements>(flow: Flow<any, any, TRequirements>) => FlowData;
|
|
1787
1690
|
/**
|
|
1788
|
-
*
|
|
1789
|
-
*
|
|
1790
|
-
* This provides type-safe storage for FlowJob metadata, tracking the
|
|
1791
|
-
* execution state of flow processing jobs.
|
|
1792
|
-
*
|
|
1793
|
-
* @example
|
|
1794
|
-
* ```typescript
|
|
1795
|
-
* const flowEffect = Effect.gen(function* () {
|
|
1796
|
-
* const jobStore = yield* FlowJobKVStore;
|
|
1797
|
-
*
|
|
1798
|
-
* // Store job state
|
|
1799
|
-
* const job: FlowJob = {
|
|
1800
|
-
* id: "job123",
|
|
1801
|
-
* flowId: "flow_resize",
|
|
1802
|
-
* status: "running",
|
|
1803
|
-
* tasks: [],
|
|
1804
|
-
* createdAt: new Date(),
|
|
1805
|
-
* updatedAt: new Date()
|
|
1806
|
-
* };
|
|
1807
|
-
* yield* jobStore.set("job123", job);
|
|
1691
|
+
* Result of a flow execution - either completed or paused.
|
|
1808
1692
|
*
|
|
1809
|
-
*
|
|
1810
|
-
* const retrieved = yield* jobStore.get("job123");
|
|
1811
|
-
* return retrieved.status;
|
|
1812
|
-
* });
|
|
1813
|
-
* ```
|
|
1814
|
-
*/
|
|
1815
|
-
declare class FlowJobKVStore extends FlowJobKVStore_base {}
|
|
1816
|
-
/**
|
|
1817
|
-
* Effect Layer that creates the FlowJobKVStore from a BaseKvStore.
|
|
1693
|
+
* @template TOutput - Type of the flow's output data
|
|
1818
1694
|
*
|
|
1819
|
-
*
|
|
1820
|
-
*
|
|
1695
|
+
* @remarks
|
|
1696
|
+
* Flows can pause when a node needs additional data (e.g., waiting for user input
|
|
1697
|
+
* or external service). The execution state allows resuming from where it paused.
|
|
1821
1698
|
*
|
|
1822
1699
|
* @example
|
|
1823
1700
|
* ```typescript
|
|
1824
|
-
* const
|
|
1825
|
-
* const jobStore = yield* FlowJobKVStore;
|
|
1826
|
-
* // Use the store...
|
|
1827
|
-
* }).pipe(
|
|
1828
|
-
* Effect.provide(flowJobKvStore),
|
|
1829
|
-
* Effect.provide(baseStoreLayer)
|
|
1830
|
-
* );
|
|
1831
|
-
* ```
|
|
1832
|
-
*/
|
|
1833
|
-
declare const flowJobKvStore: Layer.Layer<FlowJobKVStore, never, BaseKvStoreService>;
|
|
1834
|
-
//#endregion
|
|
1835
|
-
//#region src/types/data-store.d.ts
|
|
1836
|
-
/**
|
|
1837
|
-
* Options for writing data to a DataStore.
|
|
1701
|
+
* const result = await Effect.runPromise(flow.run({ inputs, storageId, jobId }));
|
|
1838
1702
|
*
|
|
1839
|
-
*
|
|
1840
|
-
*
|
|
1841
|
-
*
|
|
1703
|
+
* if (result.type === "completed") {
|
|
1704
|
+
* console.log("Flow completed:", result.result);
|
|
1705
|
+
* } else {
|
|
1706
|
+
* console.log("Flow paused at node:", result.nodeId);
|
|
1707
|
+
* // Can resume later with: flow.resume({ jobId, executionState: result.executionState, ... })
|
|
1708
|
+
* }
|
|
1709
|
+
* ```
|
|
1842
1710
|
*/
|
|
1843
|
-
type
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1711
|
+
type FlowExecutionResult<TOutput> = {
|
|
1712
|
+
type: "completed";
|
|
1713
|
+
result: TOutput;
|
|
1714
|
+
outputs?: TypedOutput[];
|
|
1715
|
+
} | {
|
|
1716
|
+
type: "paused";
|
|
1717
|
+
nodeId: string;
|
|
1718
|
+
executionState: {
|
|
1719
|
+
executionOrder: string[];
|
|
1720
|
+
currentIndex: number;
|
|
1721
|
+
inputs: Record<string, unknown>;
|
|
1722
|
+
};
|
|
1847
1723
|
};
|
|
1848
1724
|
/**
|
|
1849
|
-
*
|
|
1725
|
+
* A Flow represents a directed acyclic graph (DAG) of processing nodes.
|
|
1850
1726
|
*
|
|
1851
|
-
*
|
|
1852
|
-
*
|
|
1853
|
-
*/
|
|
1854
|
-
type UploadStrategy = "single" | "parallel";
|
|
1855
|
-
/**
|
|
1856
|
-
* Capabilities and constraints of a DataStore implementation.
|
|
1727
|
+
* Flows execute nodes in topological order, passing data between nodes through edges.
|
|
1728
|
+
* They support conditional execution, retry logic, pausable nodes, and event emission.
|
|
1857
1729
|
*
|
|
1858
|
-
*
|
|
1859
|
-
*
|
|
1860
|
-
*
|
|
1730
|
+
* @template TFlowInputSchema - Zod schema defining the shape of input data
|
|
1731
|
+
* @template TFlowOutputSchema - Zod schema defining the shape of output data
|
|
1732
|
+
* @template TRequirements - Effect requirements (services/contexts) needed by nodes
|
|
1861
1733
|
*
|
|
1862
|
-
* @property
|
|
1863
|
-
* @property
|
|
1864
|
-
* @property
|
|
1865
|
-
* @property
|
|
1866
|
-
* @property
|
|
1867
|
-
* @property
|
|
1868
|
-
* @property
|
|
1869
|
-
* @property
|
|
1870
|
-
* @property
|
|
1871
|
-
* @property
|
|
1872
|
-
* @property
|
|
1873
|
-
* @property
|
|
1874
|
-
*
|
|
1734
|
+
* @property id - Unique flow identifier
|
|
1735
|
+
* @property name - Human-readable flow name
|
|
1736
|
+
* @property nodes - Array of nodes in the flow
|
|
1737
|
+
* @property edges - Connections between nodes
|
|
1738
|
+
* @property inputSchema - Zod schema for validating flow inputs
|
|
1739
|
+
* @property outputSchema - Zod schema for validating flow outputs
|
|
1740
|
+
* @property onEvent - Optional callback for flow execution events
|
|
1741
|
+
* @property run - Executes the flow from the beginning
|
|
1742
|
+
* @property resume - Resumes a paused flow execution
|
|
1743
|
+
* @property validateTypes - Validates node type compatibility
|
|
1744
|
+
* @property validateInputs - Validates input data against schema
|
|
1745
|
+
* @property validateOutputs - Validates output data against schema
|
|
1746
|
+
*
|
|
1747
|
+
* @remarks
|
|
1748
|
+
* Flows are created using {@link createFlowWithSchema}. The Effect-based design
|
|
1749
|
+
* allows for composable error handling, resource management, and dependency injection.
|
|
1875
1750
|
*
|
|
1876
1751
|
* @example
|
|
1877
1752
|
* ```typescript
|
|
1878
|
-
* const
|
|
1753
|
+
* const flow = yield* createFlowWithSchema({
|
|
1754
|
+
* flowId: "image-pipeline",
|
|
1755
|
+
* name: "Image Processing Pipeline",
|
|
1756
|
+
* nodes: [inputNode, resizeNode, optimizeNode, storageNode],
|
|
1757
|
+
* edges: [
|
|
1758
|
+
* { source: "input", target: "resize" },
|
|
1759
|
+
* { source: "resize", target: "optimize" },
|
|
1760
|
+
* { source: "optimize", target: "storage" }
|
|
1761
|
+
* ],
|
|
1762
|
+
* inputSchema: z.object({ file: z.instanceof(File) }),
|
|
1763
|
+
* outputSchema: uploadFileSchema
|
|
1764
|
+
* });
|
|
1879
1765
|
*
|
|
1880
|
-
*
|
|
1881
|
-
*
|
|
1882
|
-
*
|
|
1883
|
-
*
|
|
1884
|
-
* }
|
|
1885
|
-
* // Use single upload
|
|
1886
|
-
* uploadAsSingleChunk(file);
|
|
1887
|
-
* }
|
|
1766
|
+
* const result = yield* flow.run({
|
|
1767
|
+
* inputs: { input: { file: myFile } },
|
|
1768
|
+
* storageId: "storage-1",
|
|
1769
|
+
* jobId: "job-123"
|
|
1770
|
+
* });
|
|
1888
1771
|
* ```
|
|
1889
1772
|
*/
|
|
1890
|
-
type
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1773
|
+
type Flow<TFlowInputSchema extends z.ZodSchema<any>, TFlowOutputSchema extends z.ZodSchema<any>, TRequirements> = {
|
|
1774
|
+
id: string;
|
|
1775
|
+
name: string;
|
|
1776
|
+
nodes: FlowNode<any, any, UploadistaError>[];
|
|
1777
|
+
edges: FlowEdge[];
|
|
1778
|
+
inputSchema: TFlowInputSchema;
|
|
1779
|
+
outputSchema: TFlowOutputSchema;
|
|
1780
|
+
onEvent?: FlowConfig<TFlowInputSchema, TFlowOutputSchema, TRequirements>["onEvent"];
|
|
1781
|
+
checkJobStatus?: FlowConfig<TFlowInputSchema, TFlowOutputSchema, TRequirements>["checkJobStatus"];
|
|
1782
|
+
hooks?: FlowConfig<TFlowInputSchema, TFlowOutputSchema, TRequirements>["hooks"];
|
|
1783
|
+
run: (args: {
|
|
1784
|
+
inputs?: Record<string, z.infer<TFlowInputSchema>>;
|
|
1785
|
+
storageId: string;
|
|
1786
|
+
jobId: string;
|
|
1787
|
+
clientId: string | null;
|
|
1788
|
+
}) => Effect.Effect<FlowExecutionResult<Record<string, z.infer<TFlowOutputSchema>>>, UploadistaError, TRequirements | UploadFileDataStores>;
|
|
1789
|
+
resume: (args: {
|
|
1790
|
+
jobId: string;
|
|
1791
|
+
storageId: string;
|
|
1792
|
+
nodeResults: Record<string, unknown>;
|
|
1793
|
+
executionState: {
|
|
1794
|
+
executionOrder: string[];
|
|
1795
|
+
currentIndex: number;
|
|
1796
|
+
inputs: Record<string, z.infer<TFlowInputSchema>>;
|
|
1797
|
+
};
|
|
1798
|
+
clientId: string | null;
|
|
1799
|
+
}) => Effect.Effect<FlowExecutionResult<Record<string, z.infer<TFlowOutputSchema>>>, UploadistaError, TRequirements | UploadFileDataStores>;
|
|
1800
|
+
validateTypes: () => {
|
|
1801
|
+
isValid: boolean;
|
|
1802
|
+
errors: string[];
|
|
1803
|
+
};
|
|
1804
|
+
validateInputs: (inputs: unknown) => {
|
|
1805
|
+
isValid: boolean;
|
|
1806
|
+
errors: string[];
|
|
1807
|
+
};
|
|
1808
|
+
validateOutputs: (outputs: unknown) => {
|
|
1809
|
+
isValid: boolean;
|
|
1810
|
+
errors: string[];
|
|
1811
|
+
};
|
|
1904
1812
|
};
|
|
1905
1813
|
/**
|
|
1906
|
-
*
|
|
1814
|
+
* Creates a new Flow with Zod schema-based type validation.
|
|
1907
1815
|
*
|
|
1908
|
-
*
|
|
1909
|
-
*
|
|
1910
|
-
*
|
|
1816
|
+
* This is the primary way to create flows in Uploadista. It constructs a Flow
|
|
1817
|
+
* instance that validates inputs/outputs, executes nodes in topological order,
|
|
1818
|
+
* handles errors with retries, and emits events during execution.
|
|
1911
1819
|
*
|
|
1912
|
-
* @template
|
|
1820
|
+
* @template TFlowInputSchema - Zod schema for flow input validation
|
|
1821
|
+
* @template TFlowOutputSchema - Zod schema for flow output validation
|
|
1822
|
+
* @template TRequirements - Effect requirements/services needed by the flow
|
|
1823
|
+
* @template TNodeError - Union of possible errors from nodes
|
|
1824
|
+
* @template TNodeRequirements - Union of requirements from nodes
|
|
1913
1825
|
*
|
|
1914
|
-
* @
|
|
1915
|
-
* @
|
|
1916
|
-
* @
|
|
1917
|
-
* @
|
|
1918
|
-
* @
|
|
1919
|
-
* @
|
|
1920
|
-
* @
|
|
1921
|
-
* @
|
|
1922
|
-
* @
|
|
1923
|
-
*
|
|
1924
|
-
* @example
|
|
1925
|
-
* ```typescript
|
|
1926
|
-
* // Implement a custom DataStore
|
|
1927
|
-
* const myDataStore: DataStore<UploadFile> = {
|
|
1928
|
-
* bucket: "my-uploads",
|
|
1929
|
-
* path: "files/",
|
|
1826
|
+
* @param config - Flow configuration object
|
|
1827
|
+
* @param config.flowId - Unique identifier for the flow
|
|
1828
|
+
* @param config.name - Human-readable flow name
|
|
1829
|
+
* @param config.nodes - Array of nodes (can be plain nodes or Effects resolving to nodes)
|
|
1830
|
+
* @param config.edges - Array of edges connecting nodes
|
|
1831
|
+
* @param config.inputSchema - Zod schema for validating inputs
|
|
1832
|
+
* @param config.outputSchema - Zod schema for validating outputs
|
|
1833
|
+
* @param config.typeChecker - Optional custom type compatibility checker
|
|
1834
|
+
* @param config.onEvent - Optional event callback for monitoring execution
|
|
1930
1835
|
*
|
|
1931
|
-
*
|
|
1932
|
-
* // Store file metadata
|
|
1933
|
-
* yield* saveMetadata(file);
|
|
1934
|
-
* return file;
|
|
1935
|
-
* }),
|
|
1836
|
+
* @returns Effect that resolves to a Flow instance
|
|
1936
1837
|
*
|
|
1937
|
-
*
|
|
1938
|
-
*
|
|
1939
|
-
*
|
|
1940
|
-
*
|
|
1941
|
-
* writeChunk(file_id, chunk, bytesWritten);
|
|
1942
|
-
* bytesWritten += chunk.byteLength;
|
|
1943
|
-
* onProgress?.(chunk.byteLength);
|
|
1944
|
-
* }));
|
|
1945
|
-
* return bytesWritten;
|
|
1946
|
-
* }),
|
|
1838
|
+
* @throws {UploadistaError} FLOW_CYCLE_ERROR if the graph contains cycles
|
|
1839
|
+
* @throws {UploadistaError} FLOW_NODE_NOT_FOUND if a node is referenced but missing
|
|
1840
|
+
* @throws {UploadistaError} FLOW_NODE_ERROR if node execution fails
|
|
1841
|
+
* @throws {UploadistaError} FLOW_OUTPUT_VALIDATION_ERROR if outputs don't match schema
|
|
1947
1842
|
*
|
|
1948
|
-
*
|
|
1949
|
-
*
|
|
1950
|
-
*
|
|
1951
|
-
*
|
|
1952
|
-
*
|
|
1843
|
+
* @remarks
|
|
1844
|
+
* - Nodes can be provided as plain objects or as Effects that resolve to nodes
|
|
1845
|
+
* - The flow performs topological sorting to determine execution order
|
|
1846
|
+
* - Conditional nodes are evaluated before execution
|
|
1847
|
+
* - Nodes can specify retry configuration with exponential backoff
|
|
1848
|
+
* - Pausable nodes can halt execution and resume later
|
|
1953
1849
|
*
|
|
1954
|
-
*
|
|
1955
|
-
*
|
|
1850
|
+
* @example
|
|
1851
|
+
* ```typescript
|
|
1852
|
+
* const flow = yield* createFlowWithSchema({
|
|
1853
|
+
* flowId: "image-upload",
|
|
1854
|
+
* name: "Image Upload with Processing",
|
|
1855
|
+
* nodes: [
|
|
1856
|
+
* inputNode,
|
|
1857
|
+
* yield* createResizeNode({ width: 1920, height: 1080 }),
|
|
1858
|
+
* optimizeNode,
|
|
1859
|
+
* storageNode
|
|
1860
|
+
* ],
|
|
1861
|
+
* edges: [
|
|
1862
|
+
* { source: "input", target: "resize" },
|
|
1863
|
+
* { source: "resize", target: "optimize" },
|
|
1864
|
+
* { source: "optimize", target: "storage" }
|
|
1865
|
+
* ],
|
|
1866
|
+
* inputSchema: z.object({
|
|
1867
|
+
* file: z.instanceof(File),
|
|
1868
|
+
* metadata: z.record(z.string(), z.any()).optional()
|
|
1956
1869
|
* }),
|
|
1870
|
+
* outputSchema: uploadFileSchema,
|
|
1871
|
+
* onEvent: (event) => Effect.gen(function* () {
|
|
1872
|
+
* console.log("Flow event:", event);
|
|
1873
|
+
* return { eventId: event.jobId };
|
|
1874
|
+
* })
|
|
1875
|
+
* });
|
|
1876
|
+
* ```
|
|
1957
1877
|
*
|
|
1958
|
-
*
|
|
1959
|
-
*
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
*
|
|
1966
|
-
* requiresOrderedChunks: false,
|
|
1967
|
-
* }),
|
|
1878
|
+
* @see {@link Flow} for the returned flow type
|
|
1879
|
+
* @see {@link FlowConfig} for configuration options
|
|
1880
|
+
*/
|
|
1881
|
+
declare function createFlowWithSchema<TFlowInputSchema extends z.ZodSchema<any>, TFlowOutputSchema extends z.ZodSchema<any>, TRequirements = never, TNodeError = never, TNodeRequirements = never>(config: FlowConfig<TFlowInputSchema, TFlowOutputSchema, TNodeError, TNodeRequirements>): Effect.Effect<Flow<TFlowInputSchema, TFlowOutputSchema, TRequirements>, TNodeError, TNodeRequirements>;
|
|
1882
|
+
//#endregion
|
|
1883
|
+
//#region src/flow/type-registry.d.ts
|
|
1884
|
+
/**
|
|
1885
|
+
* Node type category - determines where the node appears in the flow.
|
|
1968
1886
|
*
|
|
1969
|
-
*
|
|
1970
|
-
*
|
|
1971
|
-
* };
|
|
1972
|
-
* ```
|
|
1887
|
+
* - `input`: Nodes that receive data from external sources (e.g., file uploads)
|
|
1888
|
+
* - `output`: Nodes that produce final results (e.g., storage, webhooks, descriptions)
|
|
1973
1889
|
*/
|
|
1974
|
-
type
|
|
1975
|
-
readonly bucket?: string;
|
|
1976
|
-
readonly path?: string;
|
|
1977
|
-
readonly create: (file: TData) => Effect.Effect<TData, UploadistaError>;
|
|
1978
|
-
readonly remove: (file_id: string) => Effect.Effect<void, UploadistaError>;
|
|
1979
|
-
readonly read: (file_id: string) => Effect.Effect<Uint8Array, UploadistaError>;
|
|
1980
|
-
readonly write: (options: DataStoreWriteOptions, dependencies: {
|
|
1981
|
-
onProgress?: (chunkSize: number) => void;
|
|
1982
|
-
}) => Effect.Effect<number, UploadistaError>;
|
|
1983
|
-
readonly deleteExpired?: () => Effect.Effect<number, UploadistaError>;
|
|
1984
|
-
readonly getCapabilities: () => DataStoreCapabilities;
|
|
1985
|
-
readonly validateUploadStrategy: (strategy: UploadStrategy) => Effect.Effect<boolean, never>;
|
|
1986
|
-
};
|
|
1987
|
-
declare const UploadFileDataStore_base: Context.TagClass<UploadFileDataStore, "UploadFileDataStore", DataStore<UploadFile>>;
|
|
1890
|
+
type NodeTypeCategory = "input" | "output";
|
|
1988
1891
|
/**
|
|
1989
|
-
*
|
|
1892
|
+
* Defines a registered node type with its schema and metadata.
|
|
1990
1893
|
*
|
|
1991
|
-
*
|
|
1992
|
-
*
|
|
1894
|
+
* Node type definitions are registered globally and used to validate and type-narrow
|
|
1895
|
+
* flow results at runtime. Each definition includes:
|
|
1896
|
+
* - A unique identifier with versioning
|
|
1897
|
+
* - A category (input or output)
|
|
1898
|
+
* - A Zod schema for runtime validation
|
|
1899
|
+
* - A semantic version for evolution
|
|
1900
|
+
* - A human-readable description
|
|
1901
|
+
*
|
|
1902
|
+
* @template TSchema - The Zod schema type for this node's data
|
|
1903
|
+
*
|
|
1904
|
+
* @property id - Unique identifier (e.g., "storage-output-v1", "webhook-output-v1")
|
|
1905
|
+
* @property category - Whether this is an input or output node type
|
|
1906
|
+
* @property schema - Zod schema for validating data produced by this node type
|
|
1907
|
+
* @property version - Semantic version (e.g., "1.0.0") for tracking type evolution
|
|
1908
|
+
* @property description - Human-readable explanation of what this node type does
|
|
1993
1909
|
*
|
|
1994
1910
|
* @example
|
|
1995
1911
|
* ```typescript
|
|
1996
|
-
* const
|
|
1997
|
-
*
|
|
1998
|
-
*
|
|
1999
|
-
*
|
|
2000
|
-
*
|
|
1912
|
+
* const storageOutputDef: NodeTypeDefinition<z.infer<typeof uploadFileSchema>> = {
|
|
1913
|
+
* id: "storage-output-v1",
|
|
1914
|
+
* category: "output",
|
|
1915
|
+
* schema: uploadFileSchema,
|
|
1916
|
+
* version: "1.0.0",
|
|
1917
|
+
* description: "Storage output node that saves files to configured storage backend",
|
|
1918
|
+
* };
|
|
2001
1919
|
* ```
|
|
2002
1920
|
*/
|
|
2003
|
-
|
|
2004
|
-
|
|
1921
|
+
interface NodeTypeDefinition<TSchema = unknown> {
|
|
1922
|
+
id: string;
|
|
1923
|
+
category: NodeTypeCategory;
|
|
1924
|
+
schema: z.ZodSchema<TSchema>;
|
|
1925
|
+
version: string;
|
|
1926
|
+
description: string;
|
|
1927
|
+
}
|
|
2005
1928
|
/**
|
|
2006
|
-
*
|
|
1929
|
+
* Result type for validation operations.
|
|
2007
1930
|
*
|
|
2008
|
-
*
|
|
2009
|
-
|
|
1931
|
+
* @template T - The expected type on successful validation
|
|
1932
|
+
*/
|
|
1933
|
+
type ValidationResult<T> = {
|
|
1934
|
+
success: true;
|
|
1935
|
+
data: T;
|
|
1936
|
+
} | {
|
|
1937
|
+
success: false;
|
|
1938
|
+
error: UploadistaError;
|
|
1939
|
+
};
|
|
1940
|
+
/**
|
|
1941
|
+
* Central registry for node type definitions.
|
|
1942
|
+
*
|
|
1943
|
+
* The FlowTypeRegistry maintains a global registry of node types with their schemas
|
|
1944
|
+
* and metadata. It provides methods for:
|
|
1945
|
+
* - Registering new node types
|
|
1946
|
+
* - Retrieving type definitions
|
|
1947
|
+
* - Listing types by category
|
|
1948
|
+
* - Validating data against registered schemas
|
|
1949
|
+
*
|
|
1950
|
+
* The registry is immutable after registration - types cannot be modified or removed
|
|
1951
|
+
* once registered to prevent runtime errors.
|
|
1952
|
+
*
|
|
1953
|
+
* @remarks
|
|
1954
|
+
* - This is a singleton - use the exported `flowTypeRegistry` instance
|
|
1955
|
+
* - Types cannot be unregistered or modified after registration
|
|
1956
|
+
* - Duplicate type IDs are rejected
|
|
1957
|
+
* - Version strings should follow semantic versioning
|
|
2010
1958
|
*
|
|
2011
1959
|
* @example
|
|
2012
1960
|
* ```typescript
|
|
2013
|
-
*
|
|
2014
|
-
*
|
|
2015
|
-
*
|
|
2016
|
-
*
|
|
2017
|
-
*
|
|
1961
|
+
* // Register a new type
|
|
1962
|
+
* flowTypeRegistry.register({
|
|
1963
|
+
* id: "webhook-output-v1",
|
|
1964
|
+
* category: "output",
|
|
1965
|
+
* schema: webhookResponseSchema,
|
|
1966
|
+
* version: "1.0.0",
|
|
1967
|
+
* description: "HTTP webhook notification output",
|
|
2018
1968
|
* });
|
|
2019
|
-
* ```
|
|
2020
|
-
*/
|
|
2021
|
-
declare class BufferedUploadFileDataStore extends BufferedUploadFileDataStore_base {}
|
|
2022
|
-
/**
|
|
2023
|
-
* Service interface for managing multiple DataStore instances.
|
|
2024
1969
|
*
|
|
2025
|
-
*
|
|
2026
|
-
*
|
|
1970
|
+
* // Retrieve a type definition
|
|
1971
|
+
* const def = flowTypeRegistry.get("webhook-output-v1");
|
|
1972
|
+
* if (def) {
|
|
1973
|
+
* console.log(def.description);
|
|
1974
|
+
* }
|
|
2027
1975
|
*
|
|
2028
|
-
*
|
|
2029
|
-
*
|
|
1976
|
+
* // List all output types
|
|
1977
|
+
* const outputTypes = flowTypeRegistry.listByCategory("output");
|
|
1978
|
+
* console.log(outputTypes.map(t => t.id));
|
|
1979
|
+
*
|
|
1980
|
+
* // Validate data
|
|
1981
|
+
* const result = flowTypeRegistry.validate("webhook-output-v1", data);
|
|
1982
|
+
* if (result.success) {
|
|
1983
|
+
* // data is now typed according to the schema
|
|
1984
|
+
* processWebhookResponse(result.data);
|
|
1985
|
+
* }
|
|
1986
|
+
* ```
|
|
2030
1987
|
*/
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
1988
|
+
declare class FlowTypeRegistry {
|
|
1989
|
+
private readonly types;
|
|
1990
|
+
constructor();
|
|
1991
|
+
/**
|
|
1992
|
+
* Register a new node type in the registry.
|
|
1993
|
+
*
|
|
1994
|
+
* Once registered, a type cannot be modified or removed. Attempting to register
|
|
1995
|
+
* a type with a duplicate ID will throw an error.
|
|
1996
|
+
*
|
|
1997
|
+
* @template T - The TypeScript type inferred from the Zod schema
|
|
1998
|
+
* @param definition - The complete type definition including schema and metadata
|
|
1999
|
+
* @throws {UploadistaError} If a type with the same ID is already registered
|
|
2000
|
+
*
|
|
2001
|
+
* @example
|
|
2002
|
+
* ```typescript
|
|
2003
|
+
* import { z } from "zod";
|
|
2004
|
+
*
|
|
2005
|
+
* flowTypeRegistry.register({
|
|
2006
|
+
* id: "description-output-v1",
|
|
2007
|
+
* category: "output",
|
|
2008
|
+
* schema: z.object({
|
|
2009
|
+
* description: z.string(),
|
|
2010
|
+
* confidence: z.number().min(0).max(1),
|
|
2011
|
+
* tags: z.array(z.string()).optional(),
|
|
2012
|
+
* }),
|
|
2013
|
+
* version: "1.0.0",
|
|
2014
|
+
* description: "AI-generated image description with confidence score",
|
|
2015
|
+
* });
|
|
2016
|
+
* ```
|
|
2017
|
+
*/
|
|
2018
|
+
register<T>(definition: NodeTypeDefinition<T>): void;
|
|
2019
|
+
/**
|
|
2020
|
+
* Retrieve a registered type definition by its ID.
|
|
2021
|
+
*
|
|
2022
|
+
* @param id - The unique type identifier (e.g., "storage-output-v1")
|
|
2023
|
+
* @returns The type definition if found, undefined otherwise
|
|
2024
|
+
*
|
|
2025
|
+
* @example
|
|
2026
|
+
* ```typescript
|
|
2027
|
+
* const def = flowTypeRegistry.get("storage-output-v1");
|
|
2028
|
+
* if (def) {
|
|
2029
|
+
* console.log(`Found ${def.description} (v${def.version})`);
|
|
2030
|
+
* } else {
|
|
2031
|
+
* console.warn("Type not registered");
|
|
2032
|
+
* }
|
|
2033
|
+
* ```
|
|
2034
|
+
*/
|
|
2035
|
+
get(id: string): NodeTypeDefinition<unknown> | undefined;
|
|
2036
|
+
/**
|
|
2037
|
+
* List all registered types in a specific category.
|
|
2038
|
+
*
|
|
2039
|
+
* @param category - The node category to filter by ("input" or "output")
|
|
2040
|
+
* @returns Array of type definitions in the specified category
|
|
2041
|
+
*
|
|
2042
|
+
* @example
|
|
2043
|
+
* ```typescript
|
|
2044
|
+
* // List all registered output types
|
|
2045
|
+
* const outputTypes = flowTypeRegistry.listByCategory("output");
|
|
2046
|
+
* console.log("Available output types:");
|
|
2047
|
+
* for (const type of outputTypes) {
|
|
2048
|
+
* console.log(`- ${type.id}: ${type.description}`);
|
|
2049
|
+
* }
|
|
2050
|
+
* ```
|
|
2051
|
+
*/
|
|
2052
|
+
listByCategory(category: NodeTypeCategory): NodeTypeDefinition<unknown>[];
|
|
2053
|
+
/**
|
|
2054
|
+
* Validate data against a registered type's schema.
|
|
2055
|
+
*
|
|
2056
|
+
* This method performs runtime validation using the Zod schema associated with
|
|
2057
|
+
* the type. If validation succeeds, the data is returned with proper typing.
|
|
2058
|
+
* If validation fails, an UploadistaError is returned with details.
|
|
2059
|
+
*
|
|
2060
|
+
* @template T - The expected TypeScript type after validation
|
|
2061
|
+
* @param typeId - The ID of the registered type to validate against
|
|
2062
|
+
* @param data - The data to validate
|
|
2063
|
+
* @returns A result object with either the validated data or an error
|
|
2064
|
+
*
|
|
2065
|
+
* @example
|
|
2066
|
+
* ```typescript
|
|
2067
|
+
* const result = flowTypeRegistry.validate("storage-output-v1", unknownData);
|
|
2068
|
+
*
|
|
2069
|
+
* if (result.success) {
|
|
2070
|
+
* // TypeScript knows result.data is an UploadFile
|
|
2071
|
+
* console.log(`File stored at: ${result.data.url}`);
|
|
2072
|
+
* } else {
|
|
2073
|
+
* console.error(`Validation failed: ${result.error.body}`);
|
|
2074
|
+
* }
|
|
2075
|
+
* ```
|
|
2076
|
+
*/
|
|
2077
|
+
validate<T>(typeId: string, data: unknown): ValidationResult<T>;
|
|
2078
|
+
/**
|
|
2079
|
+
* Check if a type is registered.
|
|
2080
|
+
*
|
|
2081
|
+
* @param id - The unique type identifier to check
|
|
2082
|
+
* @returns True if the type is registered, false otherwise
|
|
2083
|
+
*
|
|
2084
|
+
* @example
|
|
2085
|
+
* ```typescript
|
|
2086
|
+
* if (flowTypeRegistry.has("custom-output-v1")) {
|
|
2087
|
+
* console.log("Custom output type is available");
|
|
2088
|
+
* }
|
|
2089
|
+
* ```
|
|
2090
|
+
*/
|
|
2091
|
+
has(id: string): boolean;
|
|
2092
|
+
/**
|
|
2093
|
+
* Get the total number of registered types.
|
|
2094
|
+
*
|
|
2095
|
+
* @returns The count of registered types
|
|
2096
|
+
*
|
|
2097
|
+
* @example
|
|
2098
|
+
* ```typescript
|
|
2099
|
+
* console.log(`Registry contains ${flowTypeRegistry.size()} types`);
|
|
2100
|
+
* ```
|
|
2101
|
+
*/
|
|
2102
|
+
size(): number;
|
|
2103
|
+
}
|
|
2036
2104
|
/**
|
|
2037
|
-
*
|
|
2105
|
+
* Global singleton instance of the flow type registry.
|
|
2038
2106
|
*
|
|
2039
|
-
*
|
|
2107
|
+
* Use this instance to register and access node type definitions throughout
|
|
2108
|
+
* your application. The registry is initialized once and shared globally.
|
|
2040
2109
|
*
|
|
2041
2110
|
* @example
|
|
2042
2111
|
* ```typescript
|
|
2043
|
-
*
|
|
2044
|
-
*
|
|
2045
|
-
*
|
|
2046
|
-
*
|
|
2047
|
-
*
|
|
2048
|
-
*
|
|
2112
|
+
* import { flowTypeRegistry } from "@uploadista/core/flow";
|
|
2113
|
+
*
|
|
2114
|
+
* // Register a type
|
|
2115
|
+
* flowTypeRegistry.register({
|
|
2116
|
+
* id: "my-output-v1",
|
|
2117
|
+
* category: "output",
|
|
2118
|
+
* schema: mySchema,
|
|
2119
|
+
* version: "1.0.0",
|
|
2120
|
+
* description: "My custom output type",
|
|
2049
2121
|
* });
|
|
2122
|
+
*
|
|
2123
|
+
* // Validate data
|
|
2124
|
+
* const result = flowTypeRegistry.validate("my-output-v1", data);
|
|
2050
2125
|
* ```
|
|
2051
2126
|
*/
|
|
2052
|
-
declare
|
|
2127
|
+
declare const flowTypeRegistry: FlowTypeRegistry;
|
|
2128
|
+
//#endregion
|
|
2129
|
+
//#region src/flow/node-types/index.d.ts
|
|
2053
2130
|
/**
|
|
2054
|
-
*
|
|
2131
|
+
* Type ID constants for built-in node types.
|
|
2055
2132
|
*
|
|
2056
|
-
*
|
|
2057
|
-
*
|
|
2058
|
-
* - Multiple named stores with routing
|
|
2059
|
-
* - Effect that resolves to a DataStore
|
|
2060
|
-
* - Pre-built Effect Layer
|
|
2133
|
+
* Use these constants when creating nodes with type information to ensure
|
|
2134
|
+
* consistency and avoid typos.
|
|
2061
2135
|
*
|
|
2062
2136
|
* @example
|
|
2063
2137
|
* ```typescript
|
|
2064
|
-
*
|
|
2065
|
-
* const config: DataStoreConfig = s3DataStore;
|
|
2066
|
-
*
|
|
2067
|
-
* // Multiple stores with routing
|
|
2068
|
-
* const config: DataStoreConfig = {
|
|
2069
|
-
* stores: {
|
|
2070
|
-
* "s3-prod": s3ProdStore,
|
|
2071
|
-
* "s3-dev": s3DevStore,
|
|
2072
|
-
* "local": localFileStore,
|
|
2073
|
-
* },
|
|
2074
|
-
* default: "s3-prod"
|
|
2075
|
-
* };
|
|
2138
|
+
* import { STREAMING_INPUT_TYPE_ID } from "@uploadista/core/flow";
|
|
2076
2139
|
*
|
|
2077
|
-
*
|
|
2078
|
-
*
|
|
2079
|
-
*
|
|
2080
|
-
* return s3Store(kvStore);
|
|
2140
|
+
* const inputNode = createFlowNode({
|
|
2141
|
+
* // ... other config
|
|
2142
|
+
* nodeTypeId: STREAMING_INPUT_TYPE_ID
|
|
2081
2143
|
* });
|
|
2082
|
-
*
|
|
2083
|
-
* // Pre-built Layer
|
|
2084
|
-
* const config: DataStoreConfig = Layer.succeed(UploadFileDataStores, {...});
|
|
2085
2144
|
* ```
|
|
2086
2145
|
*/
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
} | Layer.Layer<UploadFileDataStores, never, UploadFileKVStore>;
|
|
2146
|
+
declare const STORAGE_OUTPUT_TYPE_ID = "storage-output-v1";
|
|
2147
|
+
declare const OCR_OUTPUT_TYPE_ID = "ocr-output-v1";
|
|
2148
|
+
declare const IMAGE_DESCRIPTION_OUTPUT_TYPE_ID = "image-description-output-v1";
|
|
2091
2149
|
/**
|
|
2092
|
-
*
|
|
2093
|
-
*
|
|
2094
|
-
* @param config - The value to check
|
|
2095
|
-
* @returns True if the value is a DataStore
|
|
2150
|
+
* OCR output schema - structured text extraction result.
|
|
2096
2151
|
*
|
|
2097
|
-
* @
|
|
2098
|
-
*
|
|
2099
|
-
*
|
|
2100
|
-
*
|
|
2101
|
-
* }
|
|
2102
|
-
* ```
|
|
2152
|
+
* @property extractedText - The text extracted from the document
|
|
2153
|
+
* @property format - Output format (text, markdown, or JSON)
|
|
2154
|
+
* @property taskType - Type of OCR task performed
|
|
2155
|
+
* @property confidence - Optional confidence score (0-1)
|
|
2103
2156
|
*/
|
|
2104
|
-
declare const
|
|
2157
|
+
declare const ocrOutputSchema: z.ZodObject<{
|
|
2158
|
+
extractedText: z.ZodString;
|
|
2159
|
+
format: z.ZodEnum<{
|
|
2160
|
+
markdown: "markdown";
|
|
2161
|
+
plain: "plain";
|
|
2162
|
+
structured: "structured";
|
|
2163
|
+
}>;
|
|
2164
|
+
taskType: z.ZodEnum<{
|
|
2165
|
+
convertToMarkdown: "convertToMarkdown";
|
|
2166
|
+
freeOcr: "freeOcr";
|
|
2167
|
+
parseFigure: "parseFigure";
|
|
2168
|
+
locateObject: "locateObject";
|
|
2169
|
+
}>;
|
|
2170
|
+
confidence: z.ZodOptional<z.ZodNumber>;
|
|
2171
|
+
}, z.core.$strip>;
|
|
2172
|
+
type OcrOutput = z.infer<typeof ocrOutputSchema>;
|
|
2105
2173
|
/**
|
|
2106
|
-
*
|
|
2107
|
-
*
|
|
2108
|
-
* This function converts any DataStoreConfig format into a proper Effect Layer
|
|
2109
|
-
* that can be provided to the UploadFileDataStores context tag.
|
|
2110
|
-
*
|
|
2111
|
-
* It handles:
|
|
2112
|
-
* - Single DataStore: Wraps in a Layer that always returns that store
|
|
2113
|
-
* - Multiple stores: Creates routing logic with optional default
|
|
2114
|
-
* - Effect<DataStore>: Executes the Effect and wraps the result
|
|
2115
|
-
* - Layer: Returns as-is
|
|
2116
|
-
*
|
|
2117
|
-
* @param config - The DataStore configuration
|
|
2118
|
-
* @returns A Layer that provides UploadFileDataStores service
|
|
2119
|
-
*
|
|
2120
|
-
* @example
|
|
2121
|
-
* ```typescript
|
|
2122
|
-
* // Create from single store
|
|
2123
|
-
* const layer = await createDataStoreLayer(s3DataStore);
|
|
2124
|
-
*
|
|
2125
|
-
* // Create from multiple stores
|
|
2126
|
-
* const layer = await createDataStoreLayer({
|
|
2127
|
-
* stores: {
|
|
2128
|
-
* "production": s3Store,
|
|
2129
|
-
* "development": localStore,
|
|
2130
|
-
* },
|
|
2131
|
-
* default: "development"
|
|
2132
|
-
* });
|
|
2174
|
+
* Image description output schema - AI-generated image analysis result.
|
|
2133
2175
|
*
|
|
2134
|
-
*
|
|
2135
|
-
*
|
|
2136
|
-
*
|
|
2137
|
-
* const store = yield* stores.getDataStore("production", null);
|
|
2138
|
-
* return store;
|
|
2139
|
-
* }).pipe(Effect.provide(layer));
|
|
2140
|
-
* ```
|
|
2176
|
+
* @property description - Human-readable description of the image
|
|
2177
|
+
* @property confidence - Confidence score for the description (0-1)
|
|
2178
|
+
* @property metadata - Additional metadata about the description
|
|
2141
2179
|
*/
|
|
2142
|
-
declare const
|
|
2180
|
+
declare const imageDescriptionOutputSchema: z.ZodObject<{
|
|
2181
|
+
description: z.ZodString;
|
|
2182
|
+
confidence: z.ZodOptional<z.ZodNumber>;
|
|
2183
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
2184
|
+
}, z.core.$strip>;
|
|
2185
|
+
type ImageDescriptionOutput = z.infer<typeof imageDescriptionOutputSchema>;
|
|
2143
2186
|
//#endregion
|
|
2144
2187
|
//#region src/types/event-broadcaster.d.ts
|
|
2145
2188
|
/**
|
|
@@ -3126,7 +3169,7 @@ declare function createUploadServer(): Effect.Effect<{
|
|
|
3126
3169
|
getCapabilities: (storageId: string, clientId: string | null) => Effect.Effect<DataStoreCapabilities, UploadistaError, never>;
|
|
3127
3170
|
subscribeToUploadEvents: (uploadId: string, connection: WebSocketConnection) => Effect.Effect<void, UploadistaError, never>;
|
|
3128
3171
|
unsubscribeFromUploadEvents: (uploadId: string) => Effect.Effect<void, UploadistaError, never>;
|
|
3129
|
-
}, never,
|
|
3172
|
+
}, never, UploadFileDataStores | UploadFileKVStore | UploadEventEmitter | GenerateId>;
|
|
3130
3173
|
/**
|
|
3131
3174
|
* Pre-built UploadServer Effect Layer.
|
|
3132
3175
|
*
|
|
@@ -3157,7 +3200,7 @@ declare function createUploadServer(): Effect.Effect<{
|
|
|
3157
3200
|
* }).pipe(Effect.provide(fullUploadSystem));
|
|
3158
3201
|
* ```
|
|
3159
3202
|
*/
|
|
3160
|
-
declare const uploadServer: Layer.Layer<UploadServer, never,
|
|
3203
|
+
declare const uploadServer: Layer.Layer<UploadServer, never, UploadFileDataStores | UploadFileKVStore | UploadEventEmitter | GenerateId>;
|
|
3161
3204
|
//#endregion
|
|
3162
3205
|
//#region src/upload/upload-strategy-negotiator.d.ts
|
|
3163
3206
|
/**
|
|
@@ -3712,8 +3755,8 @@ declare function createFlowServer(): Effect.Effect<{
|
|
|
3712
3755
|
cancelFlow: (jobId: string, clientId: string | null) => Effect.Effect<FlowJob, UploadistaError, never>;
|
|
3713
3756
|
subscribeToFlowEvents: (jobId: string, connection: WebSocketConnection) => Effect.Effect<void, UploadistaError, never>;
|
|
3714
3757
|
unsubscribeFromFlowEvents: (jobId: string) => Effect.Effect<void, UploadistaError, never>;
|
|
3715
|
-
}, never,
|
|
3716
|
-
declare const flowServer: Layer.Layer<FlowServer, never,
|
|
3758
|
+
}, never, UploadServer | FlowEventEmitter | FlowJobKVStore | FlowProvider>;
|
|
3759
|
+
declare const flowServer: Layer.Layer<FlowServer, never, UploadServer | FlowEventEmitter | FlowJobKVStore | FlowProvider>;
|
|
3717
3760
|
type FlowServerLayer = typeof flowServer;
|
|
3718
3761
|
//#endregion
|
|
3719
3762
|
//#region src/flow/nodes/input-node.d.ts
|
|
@@ -3841,154 +3884,6 @@ declare function createInputNode(id: string, params?: InputNodeParams): Effect.E
|
|
|
3841
3884
|
type: NodeType.input;
|
|
3842
3885
|
}, UploadistaError, UploadServer>;
|
|
3843
3886
|
//#endregion
|
|
3844
|
-
//#region src/flow/nodes/storage-node.d.ts
|
|
3845
|
-
/**
|
|
3846
|
-
* Schema for storage node parameters.
|
|
3847
|
-
* Currently empty but can be extended for storage-specific configuration.
|
|
3848
|
-
*/
|
|
3849
|
-
declare const storageParamsSchema: z.ZodObject<{}, z.core.$strip>;
|
|
3850
|
-
/**
|
|
3851
|
-
* Parameters for the storage node.
|
|
3852
|
-
* Currently no parameters are required, but the schema is available for future extensions.
|
|
3853
|
-
*/
|
|
3854
|
-
type StorageParams = z.infer<typeof storageParamsSchema>;
|
|
3855
|
-
/**
|
|
3856
|
-
* Creates a storage node for storing files in the specified storage.
|
|
3857
|
-
*
|
|
3858
|
-
* The storage node handles the process of:
|
|
3859
|
-
* 1. Reading the input file from the upload server
|
|
3860
|
-
* 2. Checking if the file is already in the target storage
|
|
3861
|
-
* 3. If not, transferring the file to the target storage
|
|
3862
|
-
* 4. Applying optional post-processing
|
|
3863
|
-
* 5. Returning the final stored file
|
|
3864
|
-
*
|
|
3865
|
-
* @param id - Unique identifier for the node
|
|
3866
|
-
* @param postProcessFile - Optional function to process the file after storage
|
|
3867
|
-
* @returns An Effect that creates a flow node configured for file storage
|
|
3868
|
-
*
|
|
3869
|
-
* @example
|
|
3870
|
-
* ```typescript
|
|
3871
|
-
* // Create basic storage node
|
|
3872
|
-
* const storageNode = yield* createStorageNode("store-file");
|
|
3873
|
-
*
|
|
3874
|
-
* // Create storage node with post-processing
|
|
3875
|
-
* const storageWithProcessing = yield* createStorageNode("store-and-process", (file) => {
|
|
3876
|
-
* return Effect.succeed({
|
|
3877
|
-
* ...file,
|
|
3878
|
-
* metadata: { ...file.metadata, processed: true }
|
|
3879
|
-
* });
|
|
3880
|
-
* });
|
|
3881
|
-
* ```
|
|
3882
|
-
*/
|
|
3883
|
-
declare function createStorageNode(id: string, postProcessFile?: (file: UploadFile) => Effect.Effect<UploadFile>): Effect.Effect<FlowNodeData & {
|
|
3884
|
-
inputSchema: z.ZodType<{
|
|
3885
|
-
id: string;
|
|
3886
|
-
offset: number;
|
|
3887
|
-
storage: {
|
|
3888
|
-
id: string;
|
|
3889
|
-
type: string;
|
|
3890
|
-
path?: string | undefined;
|
|
3891
|
-
uploadId?: string | undefined;
|
|
3892
|
-
bucket?: string | undefined;
|
|
3893
|
-
parts?: {
|
|
3894
|
-
partNumber: number;
|
|
3895
|
-
etag: string;
|
|
3896
|
-
size: number;
|
|
3897
|
-
}[] | undefined;
|
|
3898
|
-
};
|
|
3899
|
-
size?: number | undefined;
|
|
3900
|
-
metadata?: Record<string, string | number | boolean> | undefined;
|
|
3901
|
-
creationDate?: string | undefined;
|
|
3902
|
-
url?: string | undefined;
|
|
3903
|
-
sizeIsDeferred?: boolean | undefined;
|
|
3904
|
-
checksum?: string | undefined;
|
|
3905
|
-
checksumAlgorithm?: string | undefined;
|
|
3906
|
-
flow?: {
|
|
3907
|
-
flowId: string;
|
|
3908
|
-
nodeId: string;
|
|
3909
|
-
jobId: string;
|
|
3910
|
-
} | undefined;
|
|
3911
|
-
}, unknown, z.core.$ZodTypeInternals<{
|
|
3912
|
-
id: string;
|
|
3913
|
-
offset: number;
|
|
3914
|
-
storage: {
|
|
3915
|
-
id: string;
|
|
3916
|
-
type: string;
|
|
3917
|
-
path?: string | undefined;
|
|
3918
|
-
uploadId?: string | undefined;
|
|
3919
|
-
bucket?: string | undefined;
|
|
3920
|
-
parts?: {
|
|
3921
|
-
partNumber: number;
|
|
3922
|
-
etag: string;
|
|
3923
|
-
size: number;
|
|
3924
|
-
}[] | undefined;
|
|
3925
|
-
};
|
|
3926
|
-
size?: number | undefined;
|
|
3927
|
-
metadata?: Record<string, string | number | boolean> | undefined;
|
|
3928
|
-
creationDate?: string | undefined;
|
|
3929
|
-
url?: string | undefined;
|
|
3930
|
-
sizeIsDeferred?: boolean | undefined;
|
|
3931
|
-
checksum?: string | undefined;
|
|
3932
|
-
checksumAlgorithm?: string | undefined;
|
|
3933
|
-
flow?: {
|
|
3934
|
-
flowId: string;
|
|
3935
|
-
nodeId: string;
|
|
3936
|
-
jobId: string;
|
|
3937
|
-
} | undefined;
|
|
3938
|
-
}, unknown>>;
|
|
3939
|
-
outputSchema: z.ZodType<UploadFile, unknown, z.core.$ZodTypeInternals<UploadFile, unknown>>;
|
|
3940
|
-
run: (args: {
|
|
3941
|
-
data: {
|
|
3942
|
-
id: string;
|
|
3943
|
-
offset: number;
|
|
3944
|
-
storage: {
|
|
3945
|
-
id: string;
|
|
3946
|
-
type: string;
|
|
3947
|
-
path?: string | undefined;
|
|
3948
|
-
uploadId?: string | undefined;
|
|
3949
|
-
bucket?: string | undefined;
|
|
3950
|
-
parts?: {
|
|
3951
|
-
partNumber: number;
|
|
3952
|
-
etag: string;
|
|
3953
|
-
size: number;
|
|
3954
|
-
}[] | undefined;
|
|
3955
|
-
};
|
|
3956
|
-
size?: number | undefined;
|
|
3957
|
-
metadata?: Record<string, string | number | boolean> | undefined;
|
|
3958
|
-
creationDate?: string | undefined;
|
|
3959
|
-
url?: string | undefined;
|
|
3960
|
-
sizeIsDeferred?: boolean | undefined;
|
|
3961
|
-
checksum?: string | undefined;
|
|
3962
|
-
checksumAlgorithm?: string | undefined;
|
|
3963
|
-
flow?: {
|
|
3964
|
-
flowId: string;
|
|
3965
|
-
nodeId: string;
|
|
3966
|
-
jobId: string;
|
|
3967
|
-
} | undefined;
|
|
3968
|
-
};
|
|
3969
|
-
jobId: string;
|
|
3970
|
-
storageId: string;
|
|
3971
|
-
flowId: string;
|
|
3972
|
-
inputs?: Record<string, unknown>;
|
|
3973
|
-
clientId: string | null;
|
|
3974
|
-
}) => Effect.Effect<NodeExecutionResult<UploadFile>, UploadistaError, never>;
|
|
3975
|
-
condition?: {
|
|
3976
|
-
field: string;
|
|
3977
|
-
operator: string;
|
|
3978
|
-
value: unknown;
|
|
3979
|
-
};
|
|
3980
|
-
multiInput?: boolean;
|
|
3981
|
-
multiOutput?: boolean;
|
|
3982
|
-
pausable?: boolean;
|
|
3983
|
-
retry?: {
|
|
3984
|
-
maxRetries?: number;
|
|
3985
|
-
retryDelay?: number;
|
|
3986
|
-
exponentialBackoff?: boolean;
|
|
3987
|
-
};
|
|
3988
|
-
} & {
|
|
3989
|
-
type: NodeType.output;
|
|
3990
|
-
}, UploadistaError, UploadServer>;
|
|
3991
|
-
//#endregion
|
|
3992
3887
|
//#region src/flow/nodes/transform-node.d.ts
|
|
3993
3888
|
/**
|
|
3994
3889
|
* Configuration object for creating a transform node.
|
|
@@ -4000,6 +3895,8 @@ interface TransformNodeConfig {
|
|
|
4000
3895
|
name: string;
|
|
4001
3896
|
/** Description of what the node does */
|
|
4002
3897
|
description: string;
|
|
3898
|
+
/** Optional node type ID for result type registration */
|
|
3899
|
+
nodeTypeId?: string;
|
|
4003
3900
|
/** Function that transforms file bytes */
|
|
4004
3901
|
transform: (bytes: Uint8Array, file: UploadFile) => Effect.Effect<Uint8Array | {
|
|
4005
3902
|
bytes: Uint8Array;
|
|
@@ -4051,6 +3948,7 @@ declare function createTransformNode({
|
|
|
4051
3948
|
id,
|
|
4052
3949
|
name,
|
|
4053
3950
|
description,
|
|
3951
|
+
nodeTypeId,
|
|
4054
3952
|
transform
|
|
4055
3953
|
}: TransformNodeConfig): Effect.Effect<FlowNodeData & {
|
|
4056
3954
|
inputSchema: zod0.ZodType<UploadFile, unknown, zod_v4_core0.$ZodTypeInternals<UploadFile, unknown>>;
|
|
@@ -5473,6 +5371,28 @@ type RemoveBackgroundParams = z.infer<typeof removeBackgroundParamsSchema>;
|
|
|
5473
5371
|
* ```
|
|
5474
5372
|
*/
|
|
5475
5373
|
declare function createTypeGuard<T>(typeId: string): (output: TypedOutput) => output is TypedOutput<T>;
|
|
5374
|
+
/**
|
|
5375
|
+
* Type guard for UploadFile objects.
|
|
5376
|
+
*
|
|
5377
|
+
* Validates that a value is a valid UploadFile by checking its structure against the schema.
|
|
5378
|
+
* This is useful for determining if a node result is an UploadFile, which affects
|
|
5379
|
+
* auto-persistence and intermediate file tracking.
|
|
5380
|
+
*
|
|
5381
|
+
* @param value - The value to check
|
|
5382
|
+
* @returns True if the value is a valid UploadFile
|
|
5383
|
+
*
|
|
5384
|
+
* @example
|
|
5385
|
+
* ```typescript
|
|
5386
|
+
* import { isUploadFile } from "@uploadista/core/flow";
|
|
5387
|
+
*
|
|
5388
|
+
* if (isUploadFile(nodeResult)) {
|
|
5389
|
+
* // nodeResult is typed as UploadFile
|
|
5390
|
+
* console.log("File ID:", nodeResult.id);
|
|
5391
|
+
* console.log("Storage:", nodeResult.storage.id);
|
|
5392
|
+
* }
|
|
5393
|
+
* ```
|
|
5394
|
+
*/
|
|
5395
|
+
declare function isUploadFile(value: unknown): value is UploadFile;
|
|
5476
5396
|
/**
|
|
5477
5397
|
* Type guard for storage output nodes.
|
|
5478
5398
|
*
|
|
@@ -5493,6 +5413,56 @@ declare function createTypeGuard<T>(typeId: string): (output: TypedOutput) => ou
|
|
|
5493
5413
|
* ```
|
|
5494
5414
|
*/
|
|
5495
5415
|
declare const isStorageOutput: (output: TypedOutput) => output is TypedOutput<UploadFile>;
|
|
5416
|
+
/**
|
|
5417
|
+
* Type guard for OCR output nodes.
|
|
5418
|
+
*
|
|
5419
|
+
* Validates that an output is from an OCR node and contains valid structured OCR data.
|
|
5420
|
+
*
|
|
5421
|
+
* @param output - The output to check
|
|
5422
|
+
* @returns True if the output is an OCR output with valid structured text data
|
|
5423
|
+
*
|
|
5424
|
+
* @example
|
|
5425
|
+
* ```typescript
|
|
5426
|
+
* import { isOcrOutput } from "@uploadista/core/flow";
|
|
5427
|
+
*
|
|
5428
|
+
* if (isOcrOutput(output)) {
|
|
5429
|
+
* // output.data is typed as OcrOutput
|
|
5430
|
+
* console.log("Extracted text:", output.data.extractedText);
|
|
5431
|
+
* console.log("Format:", output.data.format);
|
|
5432
|
+
* console.log("Task type:", output.data.taskType);
|
|
5433
|
+
* }
|
|
5434
|
+
* ```
|
|
5435
|
+
*/
|
|
5436
|
+
declare const isOcrOutput: (output: TypedOutput) => output is TypedOutput<{
|
|
5437
|
+
extractedText: string;
|
|
5438
|
+
format: "markdown" | "plain" | "structured";
|
|
5439
|
+
taskType: "convertToMarkdown" | "freeOcr" | "parseFigure" | "locateObject";
|
|
5440
|
+
confidence?: number | undefined;
|
|
5441
|
+
}>;
|
|
5442
|
+
/**
|
|
5443
|
+
* Type guard for image description output nodes.
|
|
5444
|
+
*
|
|
5445
|
+
* Validates that an output is from an image description node and contains valid description data.
|
|
5446
|
+
*
|
|
5447
|
+
* @param output - The output to check
|
|
5448
|
+
* @returns True if the output is an image description output with valid description data
|
|
5449
|
+
*
|
|
5450
|
+
* @example
|
|
5451
|
+
* ```typescript
|
|
5452
|
+
* import { isImageDescriptionOutput } from "@uploadista/core/flow";
|
|
5453
|
+
*
|
|
5454
|
+
* if (isImageDescriptionOutput(output)) {
|
|
5455
|
+
* // output.data is typed as ImageDescriptionOutput
|
|
5456
|
+
* console.log("Description:", output.data.description);
|
|
5457
|
+
* console.log("Confidence:", output.data.confidence);
|
|
5458
|
+
* }
|
|
5459
|
+
* ```
|
|
5460
|
+
*/
|
|
5461
|
+
declare const isImageDescriptionOutput: (output: TypedOutput) => output is TypedOutput<{
|
|
5462
|
+
description: string;
|
|
5463
|
+
confidence?: number | undefined;
|
|
5464
|
+
metadata?: Record<string, unknown> | undefined;
|
|
5465
|
+
}>;
|
|
5496
5466
|
/**
|
|
5497
5467
|
* Filter an array of outputs to only those matching a specific type.
|
|
5498
5468
|
*
|
|
@@ -5946,9 +5916,9 @@ type InferNode<T> = T extends FlowNode<any, any, UploadistaError> ? T : ResolveE
|
|
|
5946
5916
|
type ExtractKeysByNodeType<TNodes extends NodeDefinitionsRecord, TType extends NodeType> = { [K in keyof TNodes]: InferNode<TNodes[K]>["type"] extends TType ? K : never }[keyof TNodes];
|
|
5947
5917
|
type SchemaInfer<T> = T extends z.ZodTypeAny ? z.infer<T> : never;
|
|
5948
5918
|
type FlowInputMap<TNodes extends NodeDefinitionsRecord> = { [K in Extract<ExtractKeysByNodeType<TNodes, NodeType.input>, string>]: SchemaInfer<InferNode<TNodes[K]>["inputSchema"]> };
|
|
5949
|
-
type FlowOutputMap<TNodes extends NodeDefinitionsRecord> = { [K in Extract<
|
|
5919
|
+
type FlowOutputMap<TNodes extends NodeDefinitionsRecord> = { [K in Extract<keyof TNodes, string>]: SchemaInfer<InferNode<TNodes[K]>["outputSchema"]> };
|
|
5950
5920
|
type FlowInputUnion<TNodes extends NodeDefinitionsRecord> = { [K in Extract<ExtractKeysByNodeType<TNodes, NodeType.input>, string>]: SchemaInfer<InferNode<TNodes[K]>["inputSchema"]> }[Extract<ExtractKeysByNodeType<TNodes, NodeType.input>, string>];
|
|
5951
|
-
type FlowOutputUnion<TNodes extends NodeDefinitionsRecord> = { [K in Extract<
|
|
5921
|
+
type FlowOutputUnion<TNodes extends NodeDefinitionsRecord> = { [K in Extract<keyof TNodes, string>]: SchemaInfer<InferNode<TNodes[K]>["outputSchema"]> }[Extract<keyof TNodes, string>];
|
|
5952
5922
|
type NodeKey<TNodes extends NodeDefinitionsRecord> = Extract<keyof TNodes, string>;
|
|
5953
5923
|
type TypedFlowEdge<TNodes extends NodeDefinitionsRecord> = {
|
|
5954
5924
|
source: NodeKey<TNodes>;
|
|
@@ -6071,5 +6041,5 @@ type ResolvedUploadMetadata = {
|
|
|
6071
6041
|
};
|
|
6072
6042
|
declare function resolveUploadMetadata(metadata: FileMetadata): ResolvedUploadMetadata;
|
|
6073
6043
|
//#endregion
|
|
6074
|
-
export {
|
|
6075
|
-
//# sourceMappingURL=index-
|
|
6044
|
+
export { transcodeVideoParamsSchema as $, BaseEventEmitterService as $n, createFlowEdge as $r, SplitPdfResult as $t, isUploadFile as A, NodeType as Ai, flowServer as An, createFlowWithSchema as Ar, logoTransformSchema as At, ZipPluginLayer as B, UploadServerShape as Bn, UploadStrategy as Br, resizeParamsSchema as Bt, getFirstOutputByType as C, FlowEventNodePause as Ci, FlowServer as Cn, NodeTypeCategory as Cr, TransformationType as Ct, isImageDescriptionOutput as D, ConditionField as Di, FlowWaitUntil as Dn, Flow as Dr, contrastTransformSchema as Dt, hasOutputOfType as E, FlowEventNodeStart as Ei, FlowServerShape as En, flowTypeRegistry as Er, brightnessTransformSchema as Et, Plugin as F, NegotiatedStrategy as Fn, DataStoreConfig as Fr, textTransformSchema as Ft, VirusScanPluginLayer as G, Middleware as Gn, FlowJobKVStore as Gr, ImageAiPluginLayer as Gt, ScanMetadata as H, uploadServer as Hn, isDataStore as Hr, optimizeParamsSchema as Ht, PluginLayer as I, UploadStrategyNegotiator as In, DataStoreWriteOptions as Ir, transformImageParamsSchema as It, VideoPluginLayer as J, MiddlewareService as Jn, UploadFileKVStore as Jr, DocumentPlugin as Jt, VirusScanPluginShape as K, MiddlewareContext as Kn, KvStore as Kr, ImageAiPluginShape as Kt, ZipInput as L, UploadStrategyOptions as Ln, UploadFileDataStore as Lr, transformationSchema as Lt, removeBackgroundParamsSchema as M, getNodeData as Mi, FlowJobStatus as Mn, BufferedUploadFileDataStore as Mr, rotateTransformSchema as Mt, DescribeImageParams as N, UploadFile as Ni, FlowJobTask as Nn, DataStore as Nr, sepiaTransformSchema as Nt, isOcrOutput as O, ConditionOperator as Oi, WaitUntilCallback as On, FlowData as Or, flipTransformSchema as Ot, describeImageParamsSchema as P, uploadFileSchema as Pi, FlowJobTaskStatus as Pn, DataStoreCapabilities as Pr, sharpenTransformSchema as Pt, TranscodeVideoParams as Q, BaseEventEmitter as Qn, FlowEdge as Qr, SplitPdfParams as Qt, ZipParams as R, UploadServer as Rn, UploadFileDataStores as Rr, watermarkTransformSchema as Rt, filterOutputsByType as S, FlowEventNodeError as Si, FlowProviderShape as Sn, FlowTypeRegistry as Sr, Transformation as St, getSingleOutputByType as T, FlowEventNodeResume as Ti, FlowServerOptions as Tn, ValidationResult as Tr, blurTransformSchema as Tt, ScanResult as U, compareMimeTypes as Un, BaseKvStore as Ur, ImageAiContext as Ut, ZipPluginShape as V, createUploadServer as Vn, createDataStoreLayer as Vr, OptimizeParams as Vt, VirusScanPlugin as W, detectMimeType as Wn, BaseKvStoreService as Wr, ImageAiPlugin as Wt, TrimVideoParams as X, InputFile as Xn, jsonSerializer as Xr, DocumentPluginShape as Xt, VideoPluginShape as Y, MiddlewareServiceLive as Yn, flowJobKvStore as Yr, DocumentPluginLayer as Yt, trimVideoParamsSchema as Z, inputFileSchema as Zn, uploadFileKvStore as Zr, MergePdfParams as Zt, ExtractLayerService as _, FlowEventFlowPause as _i, InputNodeParams as _n, OCR_OUTPUT_TYPE_ID as _r, RotateTransform as _t, FlowInputMap as a, NodeConnectionValidator as ai, OcrResolution as an, flowEventEmitter as ar, describeVideoMetadataSchema as at, FlowCondition as b, FlowEventJobStart as bi, inputNodeParamsSchema as bn, imageDescriptionOutputSchema as br, TextTransform as bt, FlowRequirements as c, TypeCompatibilityChecker as ci, CredentialProvider as cn, WebSocketMessage as cr, ImagePluginShape as ct, TypedFlow as d, waitingNodeExecution as di, ExecutionLevel as dn, UploadEventType as dr, ContrastTransform as dt, BuiltInTypedOutput as ei, DocumentAiContext as en, EventEmitter as er, ResizeVideoParams as et, TypedFlowConfig as f, EventType as fi, ParallelScheduler as fn, uploadEventSchema as fr, FlipTransform as ft, ExtractEffectRequirements as g, FlowEventFlowError as gi, InputData as gn, ImageDescriptionOutput as gr, ResizeTransform as gt, ExtractEffectError as h, FlowEventFlowEnd as hi, createTransformNode as hn, IMAGE_DESCRIPTION_OUTPUT_TYPE_ID as hr, OverlayPosition as ht, runArgsSchema as i, FlowNodeData as ii, OcrParams as in, eventToMessageSerializer as ir, DescribeVideoMetadata as it, RemoveBackgroundParams as j, createFlowNode as ji, FlowJob as jn, getFlowData as jr, resizeTransformSchema as jt, isStorageOutput as k, ConditionValue as ki, createFlowServer as kn, FlowExecutionResult as kr, grayscaleTransformSchema as kt, NodeDefinition as l, TypedOutput as li, CredentialProviderLayer as ln, webSocketMessageSchema as lr, BlurTransform as lt, createFlow as m, FlowEventFlowCancel as mi, TransformNodeConfig as mn, EventBroadcasterService as mr, LogoTransform as mt, resolveUploadMetadata as n, FlowConfig as ni, DocumentAiPluginLayer as nn, TypedEventEmitter as nr, ExtractFrameVideoParams as nt, FlowOutputMap as o, NodeExecutionResult as oi, OcrResult as on, uploadEventEmitter as or, ImagePlugin as ot, TypedFlowEdge as p, FlowEvent as pi, ParallelSchedulerConfig as pn, EventBroadcaster as pr, GrayscaleTransform as pt, VideoPlugin as q, MiddlewareNext as qn, TypedKvStore as qr, DocumentMetadata as qt, RunArgs as r, FlowNode as ri, DocumentAiPluginShape as rn, UploadEventEmitter as rr, extractFrameVideoParamsSchema as rt, FlowPluginRequirements as s, NodeTypeMap as si, OcrTaskType as sn, WebSocketConnection as sr, ImagePluginLayer as st, ResolvedUploadMetadata as t, CustomTypedOutput as ti, DocumentAiPlugin as tn, FlowEventEmitter as tr, resizeVideoParamsSchema as tt, NodeDefinitionsRecord as u, completeNodeExecution as ui, CredentialProviderShape as un, UploadEvent as ur, BrightnessTransform as ut, ExtractLayerServices as v, FlowEventFlowStart as vi, createInputNode as vn, OcrOutput as vr, SepiaTransform as vt, getOutputByNodeId as w, FlowEventNodeResponse as wi, FlowServerLayer as wn, NodeTypeDefinition as wr, WatermarkTransform as wt, createTypeGuard as x, FlowEventNodeEnd as xi, FlowProvider as xn, ocrOutputSchema as xr, TransformImageParams as xt, ResolveEffect as y, FlowEventJobEnd as yi, inputDataSchema as yn, STORAGE_OUTPUT_TYPE_ID as yr, SharpenTransform as yt, ZipPlugin as z, UploadServerOptions as zn, UploadFileDataStoresShape as zr, ResizeParams as zt };
|
|
6045
|
+
//# sourceMappingURL=index-D9YSNFdA.d.cts.map
|