flowcraft 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +9 -0
- package/LICENSE +21 -0
- package/README.md +249 -0
- package/config/tsconfig.json +21 -0
- package/config/tsup.config.ts +11 -0
- package/config/vitest.config.ts +11 -0
- package/docs/.vitepress/config.ts +105 -0
- package/docs/api-reference/builder.md +158 -0
- package/docs/api-reference/fn.md +142 -0
- package/docs/api-reference/index.md +38 -0
- package/docs/api-reference/workflow.md +126 -0
- package/docs/guide/advanced-guides/cancellation.md +117 -0
- package/docs/guide/advanced-guides/composition.md +68 -0
- package/docs/guide/advanced-guides/custom-executor.md +180 -0
- package/docs/guide/advanced-guides/error-handling.md +135 -0
- package/docs/guide/advanced-guides/logging.md +106 -0
- package/docs/guide/advanced-guides/middleware.md +106 -0
- package/docs/guide/advanced-guides/observability.md +175 -0
- package/docs/guide/best-practices/debugging.md +182 -0
- package/docs/guide/best-practices/state-management.md +120 -0
- package/docs/guide/best-practices/sub-workflow-data.md +95 -0
- package/docs/guide/best-practices/testing.md +187 -0
- package/docs/guide/builders.md +157 -0
- package/docs/guide/functional-api.md +133 -0
- package/docs/guide/index.md +178 -0
- package/docs/guide/recipes/creating-a-loop.md +113 -0
- package/docs/guide/recipes/data-processing-pipeline.md +123 -0
- package/docs/guide/recipes/fan-out-fan-in.md +112 -0
- package/docs/guide/recipes/index.md +15 -0
- package/docs/guide/recipes/resilient-api-call.md +110 -0
- package/docs/guide/tooling/graph-validation.md +160 -0
- package/docs/guide/tooling/mermaid.md +156 -0
- package/docs/index.md +56 -0
- package/eslint.config.js +16 -0
- package/package.json +40 -0
- package/pnpm-workspace.yaml +2 -0
- package/sandbox/1.basic/README.md +45 -0
- package/sandbox/1.basic/package.json +16 -0
- package/sandbox/1.basic/src/flow.ts +17 -0
- package/sandbox/1.basic/src/main.ts +22 -0
- package/sandbox/1.basic/src/nodes.ts +112 -0
- package/sandbox/1.basic/src/utils.ts +35 -0
- package/sandbox/1.basic/tsconfig.json +3 -0
- package/sandbox/2.research/README.md +46 -0
- package/sandbox/2.research/package.json +16 -0
- package/sandbox/2.research/src/flow.ts +14 -0
- package/sandbox/2.research/src/main.ts +31 -0
- package/sandbox/2.research/src/nodes.ts +108 -0
- package/sandbox/2.research/src/utils.ts +45 -0
- package/sandbox/2.research/src/visualize.ts +29 -0
- package/sandbox/2.research/tsconfig.json +3 -0
- package/sandbox/3.parallel/README.md +65 -0
- package/sandbox/3.parallel/package.json +16 -0
- package/sandbox/3.parallel/src/main.ts +45 -0
- package/sandbox/3.parallel/src/nodes.ts +43 -0
- package/sandbox/3.parallel/src/utils.ts +25 -0
- package/sandbox/3.parallel/tsconfig.json +3 -0
- package/sandbox/4.dag/README.md +179 -0
- package/sandbox/4.dag/data/1.blog-post/100.json +60 -0
- package/sandbox/4.dag/data/1.blog-post/README.md +25 -0
- package/sandbox/4.dag/data/2.job-application/200.json +103 -0
- package/sandbox/4.dag/data/2.job-application/201.json +31 -0
- package/sandbox/4.dag/data/2.job-application/202.json +31 -0
- package/sandbox/4.dag/data/2.job-application/README.md +58 -0
- package/sandbox/4.dag/data/3.customer-review/300.json +141 -0
- package/sandbox/4.dag/data/3.customer-review/301.json +31 -0
- package/sandbox/4.dag/data/3.customer-review/302.json +28 -0
- package/sandbox/4.dag/data/3.customer-review/README.md +71 -0
- package/sandbox/4.dag/data/4.content-moderation/400.json +161 -0
- package/sandbox/4.dag/data/4.content-moderation/401.json +47 -0
- package/sandbox/4.dag/data/4.content-moderation/402.json +46 -0
- package/sandbox/4.dag/data/4.content-moderation/403.json +31 -0
- package/sandbox/4.dag/data/4.content-moderation/README.md +83 -0
- package/sandbox/4.dag/package.json +19 -0
- package/sandbox/4.dag/src/main.ts +73 -0
- package/sandbox/4.dag/src/nodes.ts +134 -0
- package/sandbox/4.dag/src/registry.ts +87 -0
- package/sandbox/4.dag/src/types.ts +25 -0
- package/sandbox/4.dag/src/utils.ts +42 -0
- package/sandbox/4.dag/tsconfig.json +3 -0
- package/sandbox/5.distributed/.env.example +1 -0
- package/sandbox/5.distributed/README.md +88 -0
- package/sandbox/5.distributed/data/1.blog-post/100.json +59 -0
- package/sandbox/5.distributed/data/1.blog-post/README.md +25 -0
- package/sandbox/5.distributed/data/2.job-application/200.json +103 -0
- package/sandbox/5.distributed/data/2.job-application/201.json +30 -0
- package/sandbox/5.distributed/data/2.job-application/202.json +30 -0
- package/sandbox/5.distributed/data/2.job-application/README.md +58 -0
- package/sandbox/5.distributed/data/3.customer-review/300.json +141 -0
- package/sandbox/5.distributed/data/3.customer-review/301.json +31 -0
- package/sandbox/5.distributed/data/3.customer-review/302.json +57 -0
- package/sandbox/5.distributed/data/3.customer-review/README.md +71 -0
- package/sandbox/5.distributed/data/4.content-moderation/400.json +173 -0
- package/sandbox/5.distributed/data/4.content-moderation/401.json +47 -0
- package/sandbox/5.distributed/data/4.content-moderation/402.json +46 -0
- package/sandbox/5.distributed/data/4.content-moderation/403.json +31 -0
- package/sandbox/5.distributed/data/4.content-moderation/README.md +83 -0
- package/sandbox/5.distributed/package.json +20 -0
- package/sandbox/5.distributed/src/client.ts +124 -0
- package/sandbox/5.distributed/src/executor.ts +69 -0
- package/sandbox/5.distributed/src/nodes.ts +136 -0
- package/sandbox/5.distributed/src/registry.ts +101 -0
- package/sandbox/5.distributed/src/types.ts +45 -0
- package/sandbox/5.distributed/src/utils.ts +69 -0
- package/sandbox/5.distributed/src/worker.ts +217 -0
- package/sandbox/5.distributed/tsconfig.json +3 -0
- package/sandbox/6.rag/.env.example +1 -0
- package/sandbox/6.rag/README.md +60 -0
- package/sandbox/6.rag/data/README.md +31 -0
- package/sandbox/6.rag/data/rag.json +58 -0
- package/sandbox/6.rag/documents/sample-cascade.txt +11 -0
- package/sandbox/6.rag/package.json +18 -0
- package/sandbox/6.rag/src/main.ts +52 -0
- package/sandbox/6.rag/src/nodes/GenerateEmbeddingsNode.ts +54 -0
- package/sandbox/6.rag/src/nodes/LLMProcessNode.ts +48 -0
- package/sandbox/6.rag/src/nodes/LoadAndChunkNode.ts +40 -0
- package/sandbox/6.rag/src/nodes/StoreInVectorDBNode.ts +36 -0
- package/sandbox/6.rag/src/nodes/VectorSearchNode.ts +53 -0
- package/sandbox/6.rag/src/nodes/index.ts +28 -0
- package/sandbox/6.rag/src/registry.ts +23 -0
- package/sandbox/6.rag/src/types.ts +44 -0
- package/sandbox/6.rag/src/utils.ts +77 -0
- package/sandbox/6.rag/tsconfig.json +3 -0
- package/sandbox/tsconfig.json +13 -0
- package/src/builder/collection.test.ts +287 -0
- package/src/builder/collection.ts +269 -0
- package/src/builder/graph.test.ts +406 -0
- package/src/builder/graph.ts +336 -0
- package/src/builder/graph.types.ts +104 -0
- package/src/builder/index.ts +3 -0
- package/src/context.ts +111 -0
- package/src/errors.ts +34 -0
- package/src/executor.ts +29 -0
- package/src/executors/in-memory.test.ts +93 -0
- package/src/executors/in-memory.ts +140 -0
- package/src/functions.test.ts +191 -0
- package/src/functions.ts +117 -0
- package/src/index.ts +5 -0
- package/src/logger.ts +41 -0
- package/src/types.ts +75 -0
- package/src/utils/graph.test.ts +144 -0
- package/src/utils/graph.ts +182 -0
- package/src/utils/index.ts +3 -0
- package/src/utils/mermaid.test.ts +239 -0
- package/src/utils/mermaid.ts +133 -0
- package/src/utils/sleep.ts +20 -0
- package/src/workflow.test.ts +622 -0
- package/src/workflow.ts +561 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# Recipes: Creating a Data Processing Pipeline
|
|
2
|
+
|
|
3
|
+
A common use case for a workflow is to process a piece of data through a series of transformation, validation, and storage steps. Flowcraft's fluent API on the `Node` class (`.map`, `.filter`, `.tap`, `.toContext`) is perfect for this, allowing you to define a clear, readable, and powerful pipeline.
|
|
4
|
+
|
|
5
|
+
> [!IMPORTANT]
|
|
6
|
+
> **Key Concept: Immutable Chains**
|
|
7
|
+
>
|
|
8
|
+
> Each method in the fluent chain (`.map`, `.filter`, etc.) is **immutable**. It returns a brand new `Node` instance and does **not** modify the node it was called on.
|
|
9
|
+
>
|
|
10
|
+
> You must capture the result of the chain in a variable.
|
|
11
|
+
>
|
|
12
|
+
> **Correct:**
|
|
13
|
+
>
|
|
14
|
+
> ```typescript
|
|
15
|
+
> // The entire chain is assigned to a new variable.
|
|
16
|
+
> const userProcessingPipeline = new ValueNode(rawUser)
|
|
17
|
+
> .tap(...)
|
|
18
|
+
> .map(...)
|
|
19
|
+
> ```
|
|
20
|
+
>
|
|
21
|
+
> **Incorrect:**
|
|
22
|
+
>
|
|
23
|
+
> ```typescript
|
|
24
|
+
> const startNode = new ValueNode(rawUser);
|
|
25
|
+
> // This does nothing! The new node returned by .tap() is discarded.
|
|
26
|
+
> startNode.tap(...);
|
|
27
|
+
> ```
|
|
28
|
+
|
|
29
|
+
## The Goal
|
|
30
|
+
|
|
31
|
+
Create a workflow that:
|
|
32
|
+
|
|
33
|
+
1. Starts with a raw user object.
|
|
34
|
+
2. Logs the initial object for debugging (`.tap`).
|
|
35
|
+
3. Transforms the object into a more usable format (`.map`).
|
|
36
|
+
4. Filters out inactive users (`.filter`).
|
|
37
|
+
5. Stores the final, processed data in the `Context` (`.toContext`).
|
|
38
|
+
|
|
39
|
+
```mermaid
|
|
40
|
+
graph TD
|
|
41
|
+
A[Start: Raw User] --> B["tap(log raw)"]
|
|
42
|
+
B --> C["map(transform)"]
|
|
43
|
+
C --> D{"filter(isActive)"}
|
|
44
|
+
D -- "true" --> E["toContext(save)"]
|
|
45
|
+
D -- "false" --> F[End / Discard]
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## The Implementation
|
|
49
|
+
|
|
50
|
+
We can define this entire pipeline by chaining methods, starting from a simple node that provides the initial data.
|
|
51
|
+
|
|
52
|
+
### 1. The Pipeline Definition
|
|
53
|
+
|
|
54
|
+
We'll use a `ValueNode` (a simple `Node` that just returns a value from its `exec` method) as the starting point for our chain.
|
|
55
|
+
|
|
56
|
+
```typescript
|
|
57
|
+
import { contextKey, FILTER_FAILED, Flow, Node, TypedContext } from 'flowcraft'
|
|
58
|
+
|
|
59
|
+
// A simple node that just returns a value, to start our chain
|
|
60
|
+
class ValueNode<T> extends Node<void, T> {
|
|
61
|
+
constructor(private value: T) { super() }
|
|
62
|
+
async exec(): Promise<T> { return this.value }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Define context keys for our results
|
|
66
|
+
const PROCESSED_USER = contextKey<string>('processed_user')
|
|
67
|
+
const FAILED_USER_ID = contextKey<number>('failed_user_id')
|
|
68
|
+
|
|
69
|
+
// Our raw input data
|
|
70
|
+
const rawUser = { id: 42, firstName: 'jane', lastName: 'doe', status: 'active' }
|
|
71
|
+
|
|
72
|
+
// --- The Pipeline ---
|
|
73
|
+
const userProcessingPipeline = new ValueNode(rawUser)
|
|
74
|
+
.tap(user => console.log(`[DEBUG] Processing user ID: ${user.id}`))
|
|
75
|
+
.map(user => ({
|
|
76
|
+
userId: user.id,
|
|
77
|
+
fullName: `${user.firstName.charAt(0).toUpperCase()}${user.firstName.slice(1)} ${user.lastName.toUpperCase()}`,
|
|
78
|
+
isActive: user.status === 'active',
|
|
79
|
+
}))
|
|
80
|
+
.filter(processedUser => processedUser.isActive) // <-- This is our conditional gate
|
|
81
|
+
.map(activeUser => `Welcome, ${activeUser.fullName}!`)
|
|
82
|
+
.toContext(PROCESSED_USER) // <-- This only runs if the filter passes
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### 2. Wiring the Branches
|
|
86
|
+
|
|
87
|
+
The `.filter()` method creates a branch. The `DEFAULT_ACTION` path continues the chain if the filter passes. The `FILTER_FAILED` path is taken if it fails. We need to wire up a node to handle that failure path.
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
// A node to handle the case where the filter fails
|
|
91
|
+
const handleInactiveUser = new ValueNode(rawUser)
|
|
92
|
+
.map(user => user.id)
|
|
93
|
+
.toContext(FAILED_USER_ID)
|
|
94
|
+
|
|
95
|
+
// Connect the failure path
|
|
96
|
+
userProcessingPipeline.next(handleInactiveUser, FILTER_FAILED)
|
|
97
|
+
|
|
98
|
+
const flow = new Flow(userProcessingPipeline)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### 3. Running the Pipeline
|
|
102
|
+
|
|
103
|
+
Now, let's run it. Since our user is `active`, the filter will pass.
|
|
104
|
+
|
|
105
|
+
```typescript
|
|
106
|
+
const context = new TypedContext()
|
|
107
|
+
await flow.run(context)
|
|
108
|
+
|
|
109
|
+
console.log('\n--- Active User Result ---')
|
|
110
|
+
console.log('Processed User:', context.get(PROCESSED_USER)) // "Welcome, Jane DOE!"
|
|
111
|
+
console.log('Failed User ID:', context.get(FAILED_USER_ID)) // undefined
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
If we change the input data to be an inactive user:
|
|
115
|
+
`const rawUser = { id: 99, ..., status: 'inactive' }`, the output would be:
|
|
116
|
+
|
|
117
|
+
```
|
|
118
|
+
--- Inactive User Result ---
|
|
119
|
+
Processed User: undefined
|
|
120
|
+
Failed User ID: 99
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
This demonstrates how to build a concise yet powerful data processing sequence that includes transformation, debugging, conditional logic, and state management, all in a single, readable chain.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# Recipes: Dynamic Fan-Out and Fan-In
|
|
2
|
+
|
|
3
|
+
A "fan-out, fan-in" pattern is where a workflow splits into multiple parallel branches that execute concurrently (fan-out) and then merges the results of those branches back together before proceeding (fan-in).
|
|
4
|
+
|
|
5
|
+
Flowcraft provides a dedicated `ParallelFlow` builder that makes creating this pattern simple and declarative.
|
|
6
|
+
|
|
7
|
+
## The Pattern
|
|
8
|
+
|
|
9
|
+
1. **`ParallelFlow`**: This builder takes an array of nodes that will be executed concurrently.
|
|
10
|
+
2. **Aggregation Node**: A single node is connected to the `ParallelFlow` instance. It will only run after *all* the parallel branches have completed, allowing it to "fan-in" or aggregate the results from the `Context`.
|
|
11
|
+
|
|
12
|
+
```mermaid
|
|
13
|
+
graph TD
|
|
14
|
+
A[Start] --> B(ParallelFlow)
|
|
15
|
+
subgraph "Run in Parallel"
|
|
16
|
+
B --> C[Process Branch 1]
|
|
17
|
+
B --> D[Process Branch 2]
|
|
18
|
+
B --> E[...]
|
|
19
|
+
end
|
|
20
|
+
C --> F((Aggregate Results))
|
|
21
|
+
D --> F
|
|
22
|
+
E --> F
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Example: Parallel Data Enrichment
|
|
26
|
+
|
|
27
|
+
Imagine we have a user object and we want to perform two slow, independent API calls simultaneously: one to get the user's recent activity and another to get their profile metadata.
|
|
28
|
+
|
|
29
|
+
### 1. Define Context and Nodes
|
|
30
|
+
|
|
31
|
+
We'll need keys for the initial input and the results of each parallel branch.
|
|
32
|
+
|
|
33
|
+
```typescript
|
|
34
|
+
import { contextKey, Flow, Node, ParallelFlow, TypedContext } from 'flowcraft'
|
|
35
|
+
|
|
36
|
+
// Context Keys
|
|
37
|
+
const USER_ID = contextKey<string>('user_id')
|
|
38
|
+
const ACTIVITY_DATA = contextKey<any>('activity_data')
|
|
39
|
+
const METADATA = contextKey<any>('metadata')
|
|
40
|
+
const FINAL_REPORT = contextKey<string>('final_report')
|
|
41
|
+
|
|
42
|
+
// A mock API call
|
|
43
|
+
function mockApiCall(operation: string, delay: number) {
|
|
44
|
+
console.log(`Starting API call: ${operation}...`)
|
|
45
|
+
return new Promise(resolve => setTimeout(() => {
|
|
46
|
+
console.log(`...Finished API call: ${operation}`)
|
|
47
|
+
resolve({ result: `${operation} data` })
|
|
48
|
+
}, delay))
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Nodes for each parallel task
|
|
52
|
+
const fetchActivityNode = new Node()
|
|
53
|
+
.exec(async ({ ctx }) => mockApiCall(`fetchActivity for ${ctx.get(USER_ID)}`, 100))
|
|
54
|
+
.toContext(ACTIVITY_DATA)
|
|
55
|
+
|
|
56
|
+
const fetchMetadataNode = new Node()
|
|
57
|
+
.exec(async ({ ctx }) => mockApiCall(`fetchMetadata for ${ctx.get(USER_ID)}`, 150))
|
|
58
|
+
.toContext(METADATA)
|
|
59
|
+
|
|
60
|
+
// The final aggregation node (the "fan-in" point)
|
|
61
|
+
const createReportNode = new Node()
|
|
62
|
+
.exec(async ({ ctx }) => {
|
|
63
|
+
const activity = ctx.get(ACTIVITY_DATA)
|
|
64
|
+
const metadata = ctx.get(METADATA)
|
|
65
|
+
return `Report created. Activity: ${activity.result}, Metadata: ${metadata.result}`
|
|
66
|
+
})
|
|
67
|
+
.toContext(FINAL_REPORT)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### 2. Wire the Flow with `ParallelFlow`
|
|
71
|
+
|
|
72
|
+
We create a `ParallelFlow` instance with our two API-calling nodes. Then, we connect our aggregation node to run after the parallel block is complete.
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
// Create the parallel fan-out block
|
|
76
|
+
const parallelEnrichment = new ParallelFlow([
|
|
77
|
+
fetchActivityNode,
|
|
78
|
+
fetchMetadataNode,
|
|
79
|
+
])
|
|
80
|
+
|
|
81
|
+
// After all parallel nodes are done, run the report node.
|
|
82
|
+
parallelEnrichment.next(createReportNode)
|
|
83
|
+
|
|
84
|
+
const enrichmentFlow = new Flow(parallelEnrichment)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### 3. Run the Flow
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
const context = new TypedContext([
|
|
91
|
+
[USER_ID, 'user-123']
|
|
92
|
+
])
|
|
93
|
+
|
|
94
|
+
console.time('ParallelExecution')
|
|
95
|
+
await enrichmentFlow.run(context)
|
|
96
|
+
console.timeEnd('ParallelExecution')
|
|
97
|
+
|
|
98
|
+
console.log(context.get(FINAL_REPORT))
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
The output will be:
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
Starting API call: fetchActivity for user-123...
|
|
105
|
+
Starting API call: fetchMetadata for user-123...
|
|
106
|
+
...Finished API call: fetchActivity for user-123
|
|
107
|
+
...Finished API call: fetchMetadata for user-123
|
|
108
|
+
ParallelExecution: 155.25ms
|
|
109
|
+
Report created. Activity: fetchActivity for user-123 data, Metadata: fetchMetadata for user-123 data
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Notice that the total execution time is approximately the duration of the *longest* API call (150ms), not the sum of both (~250ms). This demonstrates how the `ParallelFlow` builder provides a clean and powerful way to implement the fan-out, fan-in pattern for I/O-bound tasks. The `GraphBuilder` uses this same component internally whenever it detects this pattern in a declarative graph.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Recipes
|
|
2
|
+
|
|
3
|
+
This section provides practical, copy-paste-friendly solutions for common patterns you'll encounter when building workflows with Flowcraft.
|
|
4
|
+
|
|
5
|
+
- **[Creating a Loop](./creating-a-loop.md)**
|
|
6
|
+
- Learn how to use a cyclical graph structure to repeat actions until a condition is met.
|
|
7
|
+
|
|
8
|
+
- **[Dynamic Fan-Out and Fan-In](./fan-out-fan-in.md)**
|
|
9
|
+
- See how to split your workflow into parallel branches and merge the results.
|
|
10
|
+
|
|
11
|
+
- **[Building a Resilient API Call Node](./resilient-api-call.md)**
|
|
12
|
+
- A crucial pattern for any real-world application. Learn how to build a node that automatically retries on failure and has a fallback plan.
|
|
13
|
+
|
|
14
|
+
- **[Creating a Data Processing Pipeline](./data-processing-pipeline.md)**
|
|
15
|
+
- Discover the power of the fluent API (`.map`, `.filter`, `.tap`) to build concise and readable data transformation chains.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# Recipes: Building a Resilient API Call Node
|
|
2
|
+
|
|
3
|
+
A very common task in any workflow is calling an external API. These calls can be unreliable due to network issues or temporary service outages. This recipe shows you how to build a robust `Node` that can handle these failures gracefully using Flowcraft's built-in retry and fallback mechanisms.
|
|
4
|
+
|
|
5
|
+
## The Goal
|
|
6
|
+
|
|
7
|
+
Create a node that:
|
|
8
|
+
|
|
9
|
+
1. Calls an external API.
|
|
10
|
+
2. If the API call fails, automatically retries the call a few times.
|
|
11
|
+
3. If all retries fail, executes a fallback logic (e.g., returns a cached or default value) so the workflow can continue.
|
|
12
|
+
|
|
13
|
+
```mermaid
|
|
14
|
+
graph TD
|
|
15
|
+
A[Start] --> B{Call API}
|
|
16
|
+
B -- Success --> D[Proceed]
|
|
17
|
+
B -- Failure --> C{Retry?}
|
|
18
|
+
C -- "Yes (< 3 attempts)" --> B
|
|
19
|
+
C -- "No (all retries failed)" --> E[Execute Fallback]
|
|
20
|
+
E --> D
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## The Implementation
|
|
24
|
+
|
|
25
|
+
We will create a `FetchDataNode` that simulates calling a flaky API.
|
|
26
|
+
|
|
27
|
+
### 1. The Node Definition
|
|
28
|
+
|
|
29
|
+
We configure the node's retry behavior directly in its constructor options. The core logic is in `exec`, and the safety net is in `execFallback`.
|
|
30
|
+
|
|
31
|
+
```typescript
|
|
32
|
+
import { contextKey, Node, NodeArgs, TypedContext } from 'flowcraft'
|
|
33
|
+
|
|
34
|
+
const API_RESULT = contextKey<any>('api_result')
|
|
35
|
+
|
|
36
|
+
class FetchDataNode extends Node<void, any> {
|
|
37
|
+
private attempts = 0
|
|
38
|
+
|
|
39
|
+
constructor() {
|
|
40
|
+
// Configure the node to try a total of 3 times (1 initial + 2 retries),
|
|
41
|
+
// waiting 200ms between attempts.
|
|
42
|
+
super({ maxRetries: 3, wait: 200 })
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// The main logic: try to call the API.
|
|
46
|
+
async exec(): Promise<any> {
|
|
47
|
+
this.attempts++
|
|
48
|
+
console.log(`Attempting to call API... (attempt #${this.attempts})`)
|
|
49
|
+
|
|
50
|
+
// Simulate a failing API
|
|
51
|
+
if (this.attempts < 3) {
|
|
52
|
+
throw new Error('API service is temporarily unavailable.')
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// This will only be reached on the 3rd attempt in our simulation.
|
|
56
|
+
console.log('API call successful!')
|
|
57
|
+
return { id: 123, data: 'live data from API' }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// The fallback logic: run if all `exec` attempts fail.
|
|
61
|
+
async execFallback({ error }): Promise<any> {
|
|
62
|
+
console.error(`All API attempts failed. Final error: "${error.message}"`)
|
|
63
|
+
console.log('Returning cached/default data as a fallback.')
|
|
64
|
+
return { id: 123, data: 'cached fallback data' }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// The post-logic: store the result (from either exec or execFallback)
|
|
68
|
+
async post({ ctx, execRes }: NodeArgs<void, any>) {
|
|
69
|
+
ctx.set(API_RESULT, execRes)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### 2. Running the Resilient Node
|
|
75
|
+
|
|
76
|
+
Now, we can run this node as part of a flow. Even though the API "fails" twice, the workflow will complete successfully because of the retry and fallback logic.
|
|
77
|
+
|
|
78
|
+
```typescript
|
|
79
|
+
import { ConsoleLogger, Flow, TypedContext } from 'flowcraft'
|
|
80
|
+
|
|
81
|
+
const resilientNode = new FetchDataNode()
|
|
82
|
+
const flow = new Flow(resilientNode)
|
|
83
|
+
const context = new TypedContext()
|
|
84
|
+
|
|
85
|
+
// Use a logger to see the retry warnings
|
|
86
|
+
await flow.run(context, { logger: new ConsoleLogger() })
|
|
87
|
+
|
|
88
|
+
console.log('\nWorkflow complete.')
|
|
89
|
+
console.log('Final result in context:', context.get(API_RESULT))
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Expected Output
|
|
93
|
+
|
|
94
|
+
The output shows the node attempting the call, logging warnings for the retries, executing the fallback, and the workflow completing successfully with the fallback data.
|
|
95
|
+
|
|
96
|
+
```
|
|
97
|
+
[INFO] Running node: FetchDataNode
|
|
98
|
+
Attempting to call API... (attempt #1)
|
|
99
|
+
[WARN] Attempt 1/3 failed for FetchDataNode. Retrying...
|
|
100
|
+
Attempting to call API... (attempt #2)
|
|
101
|
+
[WARN] Attempt 2/3 failed for FetchDataNode. Retrying...
|
|
102
|
+
[ERROR] All retries failed for FetchDataNode. Executing fallback.
|
|
103
|
+
All API attempts failed. Final error: "API service is temporarily unavailable."
|
|
104
|
+
Returning cached/default data as a fallback.
|
|
105
|
+
|
|
106
|
+
Workflow complete.
|
|
107
|
+
Final result in context: { id: 123, data: 'cached fallback data' }
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
This pattern is essential for building production-grade workflows that can withstand transient failures. For more details, see the full guide on **[Error Handling](../advanced-guides/error-handling.md)**.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# Validating Workflows with Graph Analysis
|
|
2
|
+
|
|
3
|
+
As your workflows grow in complexity, it becomes crucial to ensure their structural integrity before they are ever executed. A graph with a cycle, an orphaned node, or an incorrect connection can lead to unexpected runtime behavior or infinite loops.
|
|
4
|
+
|
|
5
|
+
Flowcraft provides a powerful, lightweight, and **type-safe** utility library for performing static analysis and validation on your declarative `WorkflowGraph` definitions. This allows you to catch errors early, enforce best practices, and build more reliable systems.
|
|
6
|
+
|
|
7
|
+
## The `analyzeGraph` Utility
|
|
8
|
+
|
|
9
|
+
The foundation of the validation library is the `analyzeGraph` function. This is a pure, static utility that takes a `WorkflowGraph` object and returns a rich `GraphAnalysis` object containing pre-computed metadata about the graph's structure.
|
|
10
|
+
|
|
11
|
+
- **`nodes`**: A map of all nodes, augmented with their `inDegree` and `outDegree`.
|
|
12
|
+
- **`startNodeIds`**: An array of all node IDs with no incoming connections.
|
|
13
|
+
- **`cycles`**: An array of any cycles found, where each cycle is an array of node IDs.
|
|
14
|
+
|
|
15
|
+
```typescript
|
|
16
|
+
import { analyzeGraph } from 'flowcraft'
|
|
17
|
+
import { myGraph } from './my-workflows'
|
|
18
|
+
|
|
19
|
+
const analysis = analyzeGraph(myGraph)
|
|
20
|
+
console.log('Start Nodes:', analysis.startNodeIds)
|
|
21
|
+
console.log('Cycles Found:', analysis.cycles.length)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Writing Validation Rules
|
|
25
|
+
|
|
26
|
+
The real power comes from creating declarative validation rules that operate on this analysis data.
|
|
27
|
+
|
|
28
|
+
### `createNodeRule` Factory
|
|
29
|
+
|
|
30
|
+
This factory function is the primary tool for building custom validators. It takes three arguments:
|
|
31
|
+
|
|
32
|
+
1. `description`: A string describing the rule for clear error messages.
|
|
33
|
+
2. `filter`: A function that selects which nodes the rule should apply to (e.g., `node => node.type === 'output'`).
|
|
34
|
+
3. `check`: A function that receives a selected node and returns whether it's valid.
|
|
35
|
+
|
|
36
|
+
### Built-in Validators
|
|
37
|
+
|
|
38
|
+
Flowcraft includes a pre-built validator for the most common and critical graph error:
|
|
39
|
+
|
|
40
|
+
- `checkForCycles`: A validator that checks the analysis for any cycles and returns a `ValidationError` for each one found.
|
|
41
|
+
|
|
42
|
+
## Putting It All Together: A Custom Validator
|
|
43
|
+
|
|
44
|
+
Let's create a custom validation function for our application that enforces a set of rules.
|
|
45
|
+
|
|
46
|
+
**Example Scenario**: Our application requires that:
|
|
47
|
+
1. All graphs must be acyclic (no loops).
|
|
48
|
+
2. Nodes of type `output` must be terminal (have no outgoing connections).
|
|
49
|
+
3. Nodes of type `condition` must have exactly one input.
|
|
50
|
+
|
|
51
|
+
```typescript
|
|
52
|
+
import type { MyAppNodeTypeMap } from './my-types' // Your application's node types
|
|
53
|
+
import {
|
|
54
|
+
analyzeGraph,
|
|
55
|
+
checkForCycles,
|
|
56
|
+
createNodeRule,
|
|
57
|
+
TypedWorkflowGraph,
|
|
58
|
+
ValidationError
|
|
59
|
+
} from 'flowcraft'
|
|
60
|
+
|
|
61
|
+
// --- 1. Define the validation ruleset for our application ---
|
|
62
|
+
const myRules = [
|
|
63
|
+
// Use the built-in cycle checker
|
|
64
|
+
checkForCycles,
|
|
65
|
+
|
|
66
|
+
// Rule: 'output' nodes must be terminal.
|
|
67
|
+
createNodeRule<MyAppNodeTypeMap>(
|
|
68
|
+
'Output must be terminal',
|
|
69
|
+
node => node.type === 'output',
|
|
70
|
+
node => ({
|
|
71
|
+
valid: node.outDegree === 0,
|
|
72
|
+
message: `Output node '${node.id}' cannot have outgoing connections.`
|
|
73
|
+
})
|
|
74
|
+
),
|
|
75
|
+
|
|
76
|
+
// Rule: 'condition' nodes must have exactly one input.
|
|
77
|
+
createNodeRule<MyAppNodeTypeMap>(
|
|
78
|
+
'Condition needs one input',
|
|
79
|
+
node => node.type === 'condition',
|
|
80
|
+
node => ({
|
|
81
|
+
valid: node.inDegree === 1,
|
|
82
|
+
message: `Condition node '${node.id}' must have exactly one input, but has ${node.inDegree}.`
|
|
83
|
+
})
|
|
84
|
+
),
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The main validation function for our application.
|
|
89
|
+
*/
|
|
90
|
+
function validateMyWorkflow(graph: TypedWorkflowGraph<MyAppNodeTypeMap>): { isValid: boolean, errors: ValidationError[] } {
|
|
91
|
+
const analysis = analyzeGraph(graph)
|
|
92
|
+
// Apply every rule in the ruleset to the graph analysis
|
|
93
|
+
const errors = myRules.flatMap(rule => rule(analysis, graph))
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
isValid: errors.length === 0,
|
|
97
|
+
errors,
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Flexible API: Type-Safe vs. Untyped Validation
|
|
103
|
+
|
|
104
|
+
The validation utilities offer a flexible, overloaded API to accommodate different needs. You can work with strongly-typed graphs for maximum safety or use basic types for simplicity.
|
|
105
|
+
|
|
106
|
+
### 1. Type-Safe Validation (Recommended)
|
|
107
|
+
|
|
108
|
+
This is the most powerful way to use the library. By defining a `NodeTypeMap` and using `TypedWorkflowGraph`, you get full autocompletion and compile-time checking of your node types and their `data` payloads within your validation rules.
|
|
109
|
+
|
|
110
|
+
> [!TIP]
|
|
111
|
+
> The `createNodeRule` function is generic and integrates with `TypedWorkflowGraph`. This enables **compile-time type checking and autocompletion** even on the `data` property of your nodes.
|
|
112
|
+
|
|
113
|
+
**Example**: Let's ensure all our `api-call` nodes have a valid `retries` property.
|
|
114
|
+
|
|
115
|
+
```typescript
|
|
116
|
+
import { createNodeRule, isNodeType, TypedWorkflowGraph } from 'flowcraft'
|
|
117
|
+
|
|
118
|
+
// 1. Define your application's specific node types
|
|
119
|
+
interface MyAppNodeTypeMap {
|
|
120
|
+
'api-call': { url: string, retries: number }
|
|
121
|
+
'output': { destination: string }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Your graph is strongly typed
|
|
125
|
+
const myGraph: TypedWorkflowGraph<MyAppNodeTypeMap> = { /* ... */ }
|
|
126
|
+
|
|
127
|
+
// 2. Use the `isNodeType` helper for a clean, readable filter
|
|
128
|
+
const rule_apiRetries = createNodeRule<MyAppNodeTypeMap>(
|
|
129
|
+
'API calls must have retries',
|
|
130
|
+
// The helper creates the type guard for you!
|
|
131
|
+
isNodeType('api-call'),
|
|
132
|
+
(node) => {
|
|
133
|
+
// Because of the type guard, `node` is correctly narrowed.
|
|
134
|
+
// `node.data.retries` is fully typed and autocompletes in your IDE!
|
|
135
|
+
const valid = node.data.retries > 0
|
|
136
|
+
return { valid, message: `API call '${node.id}' must have at least 1 retry.` }
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### 2. Untyped Validation (Flexible)
|
|
142
|
+
|
|
143
|
+
If you are working with a graph where the types are dynamic or you don't have a `NodeTypeMap` available, you can use the untyped version of the API. The functions will work with the base `WorkflowGraph` and `GraphNode` types.
|
|
144
|
+
|
|
145
|
+
```typescript
|
|
146
|
+
import { createNodeRule, WorkflowGraph } from 'flowcraft'
|
|
147
|
+
|
|
148
|
+
// The graph is of the basic, untyped shape
|
|
149
|
+
const myGraph: WorkflowGraph = { /* ... */ }
|
|
150
|
+
|
|
151
|
+
const rule_noOrphans = createNodeRule(
|
|
152
|
+
'No orphaned nodes',
|
|
153
|
+
// The `node` parameter is of the base `GraphNode` type
|
|
154
|
+
_node => true, // Apply to all nodes
|
|
155
|
+
node => ({
|
|
156
|
+
valid: node.inDegree > 0 || node.outDegree > 0,
|
|
157
|
+
message: `Node '${node.id}' has no connections.`
|
|
158
|
+
})
|
|
159
|
+
)
|
|
160
|
+
```
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Visualizing Workflows with Mermaid.js
|
|
2
|
+
|
|
3
|
+
Complex workflows with multiple branches, loops, and fan-outs can be difficult to reason about from code alone. To help with this, Flowcraft includes a `generateMermaidGraph` utility that can automatically create a visual diagram of any `Flow` instance.
|
|
4
|
+
|
|
5
|
+
This utility is an invaluable tool for:
|
|
6
|
+
|
|
7
|
+
- **Debugging**: Quickly verify that your nodes are wired together exactly as you intended.
|
|
8
|
+
- **Documentation**: Embed diagrams directly into your project's `README.md` or technical documentation.
|
|
9
|
+
- **Onboarding**: Help new team members understand the control flow of a complex business process at a glance.
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
To use the utility, simply import `generateMermaidGraph`, create your `Flow` instance, and pass it to the function.
|
|
14
|
+
|
|
15
|
+
Let's visualize the "Research Agent" from the sandbox examples, which contains a decision loop:
|
|
16
|
+
|
|
17
|
+
```typescript
|
|
18
|
+
import { DEFAULT_ACTION, Flow, generateMermaidGraph, Node } from 'flowcraft'
|
|
19
|
+
|
|
20
|
+
// Define the nodes for the agent
|
|
21
|
+
class DecideActionNode extends Node<void, void, 'search' | 'answer'> {
|
|
22
|
+
async post() { return 'search' /* or 'answer' */ }
|
|
23
|
+
}
|
|
24
|
+
class SearchWebNode extends Node {}
|
|
25
|
+
class AnswerQuestionNode extends Node {}
|
|
26
|
+
|
|
27
|
+
// Create instances
|
|
28
|
+
const decideNode = new DecideActionNode()
|
|
29
|
+
const searchNode = new SearchWebNode()
|
|
30
|
+
const answerNode = new AnswerQuestionNode()
|
|
31
|
+
|
|
32
|
+
// Wire the graph
|
|
33
|
+
decideNode.next(searchNode, 'search')
|
|
34
|
+
decideNode.next(answerNode, 'answer')
|
|
35
|
+
searchNode.next(decideNode, DEFAULT_ACTION) // Loop back to the decision node
|
|
36
|
+
|
|
37
|
+
const researchAgentFlow = new Flow(decideNode)
|
|
38
|
+
|
|
39
|
+
// Generate and print the Mermaid syntax
|
|
40
|
+
const mermaidGraph = generateMermaidGraph(researchAgentFlow)
|
|
41
|
+
console.log(mermaidGraph)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Generated Syntax
|
|
45
|
+
|
|
46
|
+
Running the code above will print the following Mermaid.js syntax to the console:
|
|
47
|
+
|
|
48
|
+
```mmd
|
|
49
|
+
graph TD
|
|
50
|
+
DecideActionNode_0[DecideActionNode]
|
|
51
|
+
SearchWebNode_0[SearchWebNode]
|
|
52
|
+
AnswerQuestionNode_0[AnswerQuestionNode]
|
|
53
|
+
DecideActionNode_0 -- "search" --> SearchWebNode_0
|
|
54
|
+
DecideActionNode_0 -- "answer" --> AnswerQuestionNode_0
|
|
55
|
+
SearchWebNode_0 --> DecideActionNode_0
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Rendered Graph
|
|
59
|
+
|
|
60
|
+
When this syntax is rendered, it produces the following diagram:
|
|
61
|
+
|
|
62
|
+
```mermaid
|
|
63
|
+
graph TD
|
|
64
|
+
DecideActionNode_0[DecideActionNode]
|
|
65
|
+
SearchWebNode_0[SearchWebNode]
|
|
66
|
+
AnswerQuestionNode_0[AnswerQuestionNode]
|
|
67
|
+
DecideActionNode_0 -- "search" --> SearchWebNode_0
|
|
68
|
+
DecideActionNode_0 -- "answer" --> AnswerQuestionNode_0
|
|
69
|
+
SearchWebNode_0 --> DecideActionNode_0
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## How to Render the Graph
|
|
73
|
+
|
|
74
|
+
You can render the generated syntax in several places:
|
|
75
|
+
|
|
76
|
+
- **GitHub**: Directly in markdown files (`.md`), issues, and pull requests.
|
|
77
|
+
- **VS Code**: Using a markdown previewer with a Mermaid extension installed.
|
|
78
|
+
- **Online Editors**: Paste the syntax into the [Mermaid Live Editor](https://mermaid.live).
|
|
79
|
+
|
|
80
|
+
## Supported Features
|
|
81
|
+
|
|
82
|
+
The visualizer correctly represents all of Flowcraft's core branching and flow control patterns.
|
|
83
|
+
|
|
84
|
+
### Conditional Branching
|
|
85
|
+
|
|
86
|
+
Custom action strings are rendered as labels on the connecting arrows.
|
|
87
|
+
|
|
88
|
+
**Code**:
|
|
89
|
+
|
|
90
|
+
```typescript
|
|
91
|
+
const decision = new DecisionNode()
|
|
92
|
+
decision.next(new PathANode(), 'path_a')
|
|
93
|
+
decision.next(new PathBNode(), 'path_b')
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
**Graph**:
|
|
97
|
+
|
|
98
|
+
```mermaid
|
|
99
|
+
graph TD
|
|
100
|
+
DecisionNode_0[DecisionNode]
|
|
101
|
+
DecisionNode_0 -- "path_a" --> PathANode_0[PathANode]
|
|
102
|
+
DecisionNode_0 -- "path_b" --> PathBNode_0[PathBNode]
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Filter Logic
|
|
106
|
+
|
|
107
|
+
The special `FILTER_FAILED` action is given a descriptive label. The `DEFAULT_ACTION` has no label for clarity.
|
|
108
|
+
|
|
109
|
+
**Code**:
|
|
110
|
+
|
|
111
|
+
```typescript
|
|
112
|
+
const filter = new FilterNode()
|
|
113
|
+
filter.next(new SuccessNode(), DEFAULT_ACTION)
|
|
114
|
+
filter.next(new FailureNode(), FILTER_FAILED)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
**Graph**:
|
|
118
|
+
|
|
119
|
+
```mermaid
|
|
120
|
+
graph TD
|
|
121
|
+
FilterNode_0[FilterNode]
|
|
122
|
+
FilterNode_0 --> SuccessNode_0[SuccessNode]
|
|
123
|
+
FilterNode_0 -- "filter failed" --> FailureNode_0[FailureNode]
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### Fan-In / Convergence
|
|
127
|
+
|
|
128
|
+
When multiple nodes connect to the same successor, the graph shows all arrows converging.
|
|
129
|
+
|
|
130
|
+
**Code**:
|
|
131
|
+
|
|
132
|
+
```typescript
|
|
133
|
+
const branchA = new PathANode()
|
|
134
|
+
const branchB = new PathBNode()
|
|
135
|
+
const end = new EndNode()
|
|
136
|
+
branchA.next(end)
|
|
137
|
+
branchB.next(end)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
**Graph**:
|
|
141
|
+
|
|
142
|
+
```mermaid
|
|
143
|
+
graph TD
|
|
144
|
+
subgraph Fan-Out
|
|
145
|
+
StartNode --> PathANode
|
|
146
|
+
StartNode --> PathBNode
|
|
147
|
+
end
|
|
148
|
+
subgraph Fan-In
|
|
149
|
+
PathANode --> EndNode
|
|
150
|
+
PathBNode --> EndNode
|
|
151
|
+
end
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### Node Naming
|
|
155
|
+
|
|
156
|
+
If your flow uses multiple instances of the same `Node` class, the generator will append a unique index to each one (e.g., `ProcessNode_0`, `ProcessNode_1`) to distinguish them in the graph.
|