@workglow/task-graph 0.0.87 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +299 -1383
- package/dist/browser.js.map +11 -19
- package/dist/bun.js +299 -1383
- package/dist/bun.js.map +11 -19
- package/dist/node.js +299 -1383
- package/dist/node.js.map +11 -19
- package/dist/task/GraphAsTaskRunner.d.ts +3 -4
- package/dist/task/GraphAsTaskRunner.d.ts.map +1 -1
- package/dist/task/index.d.ts +1 -13
- package/dist/task/index.d.ts.map +1 -1
- package/dist/task-graph/ITaskGraph.d.ts +1 -1
- package/dist/task-graph/ITaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraph.d.ts +1 -1
- package/dist/task-graph/TaskGraph.d.ts.map +1 -1
- package/dist/task-graph/TaskGraphRunner.d.ts +2 -1
- package/dist/task-graph/TaskGraphRunner.d.ts.map +1 -1
- package/dist/task-graph/Workflow.d.ts +8 -94
- package/dist/task-graph/Workflow.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/EXECUTION_MODEL.md +433 -0
- package/dist/task/BatchTask.d.ts +0 -154
- package/dist/task/BatchTask.d.ts.map +0 -1
- package/dist/task/ForEachTask.d.ts +0 -120
- package/dist/task/ForEachTask.d.ts.map +0 -1
- package/dist/task/IteratorTask.d.ts +0 -197
- package/dist/task/IteratorTask.d.ts.map +0 -1
- package/dist/task/IteratorTaskRunner.d.ts +0 -63
- package/dist/task/IteratorTaskRunner.d.ts.map +0 -1
- package/dist/task/MapTask.d.ts +0 -134
- package/dist/task/MapTask.d.ts.map +0 -1
- package/dist/task/ReduceTask.d.ts +0 -155
- package/dist/task/ReduceTask.d.ts.map +0 -1
- package/dist/task/WhileTask.d.ts +0 -176
- package/dist/task/WhileTask.d.ts.map +0 -1
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
# Task Graph Execution Model
|
|
2
|
+
|
|
3
|
+
This document explains the internal execution model of the task graph system. It is intended for developers (human or AI) who need to understand or modify the execution logic.
|
|
4
|
+
|
|
5
|
+
## Table of Contents
|
|
6
|
+
|
|
7
|
+
- [Overview](#overview)
|
|
8
|
+
- [Task Lifecycle](#task-lifecycle)
|
|
9
|
+
- [Normal Execution (run)](#normal-execution-run)
|
|
10
|
+
- [Reactive Execution (runReactive)](#reactive-execution-runreactive)
|
|
11
|
+
- [Dataflow and Input Propagation](#dataflow-and-input-propagation)
|
|
12
|
+
- [GraphAsTask (Subgraphs)](#graphastask-subgraphs)
|
|
13
|
+
- [Key Invariants](#key-invariants)
|
|
14
|
+
- [Common Pitfalls](#common-pitfalls)
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## Overview
|
|
19
|
+
|
|
20
|
+
The task graph system has two execution modes:
|
|
21
|
+
|
|
22
|
+
1. **`run()`** - Full execution that produces cached, immutable results
|
|
23
|
+
2. **`runReactive()`** - Lightweight execution for UI updates and previews
|
|
24
|
+
|
|
25
|
+
These modes serve different purposes and have different semantics regarding task state and data flow.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Task Lifecycle
|
|
30
|
+
|
|
31
|
+
### Task Statuses
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
PENDING → PROCESSING → COMPLETED
|
|
35
|
+
↘ FAILED
|
|
36
|
+
↘ ABORTED
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
| Status | Description |
|
|
40
|
+
| ------------ | ------------------------------------------------------------------- |
|
|
41
|
+
| `PENDING` | Task has not been executed yet. Inputs can be modified. |
|
|
42
|
+
| `PROCESSING` | Task is currently executing. |
|
|
43
|
+
| `COMPLETED` | Task has finished successfully. **Output is locked and immutable.** |
|
|
44
|
+
| `FAILED` | Task execution threw an error. |
|
|
45
|
+
| `ABORTED` | Task was cancelled via `abort()`. |
|
|
46
|
+
|
|
47
|
+
### Key Properties
|
|
48
|
+
|
|
49
|
+
Each task maintains:
|
|
50
|
+
|
|
51
|
+
- `defaults` - Default input values set at construction time
|
|
52
|
+
- `runInputData` - The actual input used for execution (defaults + overrides)
|
|
53
|
+
- `runOutputData` - The output produced by execution
|
|
54
|
+
- `status` - Current lifecycle status
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Normal Execution (run)
|
|
59
|
+
|
|
60
|
+
### Purpose
|
|
61
|
+
|
|
62
|
+
Full execution that:
|
|
63
|
+
|
|
64
|
+
- Runs the task's `execute()` method
|
|
65
|
+
- Produces cached, deterministic results
|
|
66
|
+
- Transitions task to `COMPLETED` status
|
|
67
|
+
- Makes output immutable
|
|
68
|
+
|
|
69
|
+
### Flow
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
Task.run(overrides)
|
|
73
|
+
↓
|
|
74
|
+
TaskRunner.run(overrides)
|
|
75
|
+
↓
|
|
76
|
+
1. setInput(overrides) # Merge overrides into runInputData
|
|
77
|
+
2. resolveSchemaInputs() # Resolve model/repository strings to instances
|
|
78
|
+
3. validateInput() # Validate against input schema
|
|
79
|
+
4. Check cache # If cacheable, check for cached result
|
|
80
|
+
5. executeTask() # Call task.execute(input, context)
|
|
81
|
+
6. Store in cache # If cacheable, cache the result
|
|
82
|
+
7. handleComplete() # Set status = COMPLETED
|
|
83
|
+
↓
|
|
84
|
+
Return runOutputData (locked)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Graph-Level Execution
|
|
88
|
+
|
|
89
|
+
```
|
|
90
|
+
TaskGraph.run(input)
|
|
91
|
+
↓
|
|
92
|
+
TaskGraphRunner.runGraph(input)
|
|
93
|
+
↓
|
|
94
|
+
For each task (in topological order):
|
|
95
|
+
1. copyInputFromEdgesToNode() # Pull data from incoming dataflows
|
|
96
|
+
2. runTask(task, input) # Execute the task
|
|
97
|
+
3. pushOutputFromNodeToEdges() # Push output to outgoing dataflows
|
|
98
|
+
↓
|
|
99
|
+
Return results from ending nodes (no outgoing dataflows)
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## Reactive Execution (runReactive)
|
|
105
|
+
|
|
106
|
+
### Purpose
|
|
107
|
+
|
|
108
|
+
Lightweight execution for:
|
|
109
|
+
|
|
110
|
+
- UI previews and updates
|
|
111
|
+
- Fast transformations (e.g., image filters)
|
|
112
|
+
- Propagating intermediate results through PENDING tasks
|
|
113
|
+
|
|
114
|
+
**Important:** Reactive execution only affects `PENDING` tasks. `COMPLETED` tasks return their cached output unchanged.
|
|
115
|
+
|
|
116
|
+
### Use Case Example
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
User edits an InputNode default → Task is PENDING
|
|
120
|
+
↓
|
|
121
|
+
runReactive() is called
|
|
122
|
+
↓
|
|
123
|
+
InputTask (PENDING) receives new value
|
|
124
|
+
↓
|
|
125
|
+
Downstream tasks (PENDING) get reactive updates
|
|
126
|
+
↓
|
|
127
|
+
Tasks run their executeReactive() for quick previews
|
|
128
|
+
↓
|
|
129
|
+
Eventually run() is called → All tasks become COMPLETED (locked)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
### Task-Level Flow
|
|
133
|
+
|
|
134
|
+
```
|
|
135
|
+
Task.runReactive(overrides)
|
|
136
|
+
↓
|
|
137
|
+
TaskRunner.runReactive(overrides)
|
|
138
|
+
↓
|
|
139
|
+
1. If status == PROCESSING: return existing output (no re-entry)
|
|
140
|
+
2. setInput(overrides) # Update runInputData
|
|
141
|
+
3. resolveSchemaInputs() # Resolve strings to instances
|
|
142
|
+
4. handleStartReactive() # Status → PROCESSING
|
|
143
|
+
5. validateInput()
|
|
144
|
+
6. executeTaskReactive(input, output) # Call task.executeReactive()
|
|
145
|
+
7. runOutputData = merge(output, result) # Merge with previous output
|
|
146
|
+
8. handleCompleteReactive() # Status → back to previous
|
|
147
|
+
↓
|
|
148
|
+
Return runOutputData
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Graph-Level Flow
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
TaskGraph.runReactive(input)
|
|
155
|
+
↓
|
|
156
|
+
TaskGraphRunner.runGraphReactive(input)
|
|
157
|
+
↓
|
|
158
|
+
For each task (in topological order):
|
|
159
|
+
↓
|
|
160
|
+
┌─ If status == PENDING:
|
|
161
|
+
│ resetInputData() # Reset to defaults
|
|
162
|
+
│ copyInputFromEdgesToNode() # Pull from incoming dataflows
|
|
163
|
+
└─ Else (COMPLETED):
|
|
164
|
+
Skip input modification # Output is locked
|
|
165
|
+
↓
|
|
166
|
+
If isRootTask (no incoming dataflows):
|
|
167
|
+
taskInput = input # Pass graph input to root tasks
|
|
168
|
+
Else:
|
|
169
|
+
taskInput = {}
|
|
170
|
+
↓
|
|
171
|
+
task.runReactive(taskInput)
|
|
172
|
+
↓
|
|
173
|
+
pushOutputFromNodeToEdges() # Push output to dataflows
|
|
174
|
+
↓
|
|
175
|
+
Return results from ending nodes
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
### The executeReactive Method
|
|
179
|
+
|
|
180
|
+
```typescript
|
|
181
|
+
// Default implementation - just returns existing output
|
|
182
|
+
async executeReactive(input, output, context): Promise<Output | undefined> {
|
|
183
|
+
return output;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Custom implementation for quick transformations
|
|
187
|
+
async executeReactive(input, output, context): Promise<Output | undefined> {
|
|
188
|
+
// Lightweight operation (e.g., < 1ms)
|
|
189
|
+
return { ...output, preview: this.quickTransform(input) };
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## Dataflow and Input Propagation
|
|
196
|
+
|
|
197
|
+
### How Data Flows Between Tasks
|
|
198
|
+
|
|
199
|
+
```
|
|
200
|
+
TaskA (source) TaskB (target)
|
|
201
|
+
↓ ↓
|
|
202
|
+
outputSchema: { inputSchema: {
|
|
203
|
+
result: number value: number
|
|
204
|
+
} }
|
|
205
|
+
↓ ↓
|
|
206
|
+
Dataflow("taskA", "result", "taskB", "value")
|
|
207
|
+
↓
|
|
208
|
+
TaskA.runOutputData.result → TaskB.runInputData.value
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
### Key Methods
|
|
212
|
+
|
|
213
|
+
| Method | Purpose |
|
|
214
|
+
| ----------------------------------------- | ---------------------------------------------------------------- |
|
|
215
|
+
| `copyInputFromEdgesToNode(task)` | Pull data from all incoming dataflows into task's `runInputData` |
|
|
216
|
+
| `pushOutputFromNodeToEdges(task, output)` | Push task's output to all outgoing dataflows |
|
|
217
|
+
| `addInputData(task, data)` | Merge data into task's `runInputData` |
|
|
218
|
+
|
|
219
|
+
### When Input is Copied
|
|
220
|
+
|
|
221
|
+
| Execution Mode | Task Status | Input Copied? |
|
|
222
|
+
| --------------- | ----------- | ------------------------- |
|
|
223
|
+
| `run()` | Any | Yes (always) |
|
|
224
|
+
| `runReactive()` | PENDING | Yes |
|
|
225
|
+
| `runReactive()` | COMPLETED | **No** (output is locked) |
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## GraphAsTask (Subgraphs)
|
|
230
|
+
|
|
231
|
+
### What is GraphAsTask?
|
|
232
|
+
|
|
233
|
+
A `GraphAsTask` is a task that contains an internal `TaskGraph` (subgraph). This enables:
|
|
234
|
+
|
|
235
|
+
- Hierarchical workflow composition
|
|
236
|
+
- Encapsulation of complex logic
|
|
237
|
+
- Reusable workflow components
|
|
238
|
+
|
|
239
|
+
### Execution Flow
|
|
240
|
+
|
|
241
|
+
```
|
|
242
|
+
GraphAsTask.run(input)
|
|
243
|
+
↓
|
|
244
|
+
GraphAsTaskRunner.executeTask(input)
|
|
245
|
+
↓
|
|
246
|
+
executeTaskChildren(input)
|
|
247
|
+
↓
|
|
248
|
+
subGraph.run(input) # Execute the entire subgraph
|
|
249
|
+
↓
|
|
250
|
+
mergeExecuteOutputsToRunOutput() # Combine results from ending nodes
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
### Reactive Execution with Subgraphs
|
|
254
|
+
|
|
255
|
+
```
|
|
256
|
+
GraphAsTask.runReactive(input)
|
|
257
|
+
↓
|
|
258
|
+
GraphAsTaskRunner.executeTaskReactive(input, output)
|
|
259
|
+
↓
|
|
260
|
+
executeTaskChildrenReactive()
|
|
261
|
+
↓
|
|
262
|
+
subGraph.runReactive(this.task.runInputData) # ← IMPORTANT: Pass parent's input
|
|
263
|
+
↓
|
|
264
|
+
mergeExecuteOutputsToRunOutput()
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
**Critical:** The parent's `runInputData` is passed to `subGraph.runReactive()` so that root tasks in the subgraph (like InputTask) receive the input values.
|
|
268
|
+
|
|
269
|
+
### Root Task Input Propagation
|
|
270
|
+
|
|
271
|
+
In `runGraphReactive()`:
|
|
272
|
+
|
|
273
|
+
```typescript
|
|
274
|
+
const isRootTask = this.graph.getSourceDataflows(task.config.id).length === 0;
|
|
275
|
+
|
|
276
|
+
// For root tasks, pass the input parameter (from parent GraphAsTask)
|
|
277
|
+
const taskInput = isRootTask ? input : {};
|
|
278
|
+
|
|
279
|
+
const taskResult = await task.runReactive(taskInput);
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
This ensures:
|
|
283
|
+
|
|
284
|
+
1. Root tasks (no incoming dataflows) receive input from the parent
|
|
285
|
+
2. Non-root tasks receive input from their upstream dataflows
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## Key Invariants
|
|
290
|
+
|
|
291
|
+
### 1. COMPLETED Tasks Are Immutable
|
|
292
|
+
|
|
293
|
+
Once a task's `run()` completes and status becomes `COMPLETED`:
|
|
294
|
+
|
|
295
|
+
- `runOutputData` is **locked** and **cacheable**
|
|
296
|
+
- `runInputData` should not be modified
|
|
297
|
+
- `runReactive()` returns the cached output unchanged
|
|
298
|
+
|
|
299
|
+
### 2. Only PENDING Tasks Receive Dataflow Updates in Reactive Mode
|
|
300
|
+
|
|
301
|
+
```typescript
|
|
302
|
+
if (task.status === TaskStatus.PENDING) {
|
|
303
|
+
task.resetInputData();
|
|
304
|
+
this.copyInputFromEdgesToNode(task);
|
|
305
|
+
}
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
### 3. Root Tasks Receive Parent Input
|
|
309
|
+
|
|
310
|
+
In subgraphs, root tasks (no incoming dataflows) receive the parent's input:
|
|
311
|
+
|
|
312
|
+
```typescript
|
|
313
|
+
const taskInput = isRootTask ? input : {};
|
|
314
|
+
task.runReactive(taskInput);
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
### 4. executeReactive is Lightweight
|
|
318
|
+
|
|
319
|
+
The `executeReactive()` method should:
|
|
320
|
+
|
|
321
|
+
- Complete quickly (< 1ms ideally)
|
|
322
|
+
- Not perform heavy computation
|
|
323
|
+
- Return UI preview data
|
|
324
|
+
|
|
325
|
+
Heavy computation belongs in `execute()`.
|
|
326
|
+
|
|
327
|
+
### 5. Reactive Execution Respects Task Order
|
|
328
|
+
|
|
329
|
+
Tasks are executed in topological order (via `reactiveScheduler`), ensuring:
|
|
330
|
+
|
|
331
|
+
- Upstream tasks run before downstream tasks
|
|
332
|
+
- Data is available when needed
|
|
333
|
+
|
|
334
|
+
---
|
|
335
|
+
|
|
336
|
+
## Common Pitfalls
|
|
337
|
+
|
|
338
|
+
### 1. Modifying COMPLETED Task Input
|
|
339
|
+
|
|
340
|
+
**Wrong:**
|
|
341
|
+
|
|
342
|
+
```typescript
|
|
343
|
+
// Trying to update a COMPLETED task's input
|
|
344
|
+
task.setInput({ newValue: 42 }); // ❌ Violates immutability
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
**Correct:**
|
|
348
|
+
Only modify input for PENDING tasks, or reset the entire graph first.
|
|
349
|
+
|
|
350
|
+
### 2. Missing Root Task Input Propagation
|
|
351
|
+
|
|
352
|
+
**Wrong:**
|
|
353
|
+
|
|
354
|
+
```typescript
|
|
355
|
+
protected async executeTaskChildrenReactive() {
|
|
356
|
+
return this.task.subGraph!.runReactive(); // ❌ No input passed
|
|
357
|
+
}
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
**Correct:**
|
|
361
|
+
|
|
362
|
+
```typescript
|
|
363
|
+
protected async executeTaskChildrenReactive() {
|
|
364
|
+
return this.task.subGraph!.runReactive(this.task.runInputData); // ✓
|
|
365
|
+
}
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
### 3. Copying Input to COMPLETED Tasks
|
|
369
|
+
|
|
370
|
+
**Wrong:**
|
|
371
|
+
|
|
372
|
+
```typescript
|
|
373
|
+
// In runGraphReactive
|
|
374
|
+
this.copyInputFromEdgesToNode(task); // ❌ Always copies, even for COMPLETED
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
**Correct:**
|
|
378
|
+
|
|
379
|
+
```typescript
|
|
380
|
+
if (task.status === TaskStatus.PENDING) {
|
|
381
|
+
task.resetInputData();
|
|
382
|
+
this.copyInputFromEdgesToNode(task); // ✓ Only for PENDING
|
|
383
|
+
}
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
### 4. Heavy Computation in executeReactive
|
|
387
|
+
|
|
388
|
+
**Wrong:**
|
|
389
|
+
|
|
390
|
+
```typescript
|
|
391
|
+
async executeReactive(input, output) {
|
|
392
|
+
// ❌ Takes 30 seconds
|
|
393
|
+
const result = await this.trainNeuralNetwork(input);
|
|
394
|
+
return { result };
|
|
395
|
+
}
|
|
396
|
+
```
|
|
397
|
+
|
|
398
|
+
**Correct:**
|
|
399
|
+
|
|
400
|
+
```typescript
|
|
401
|
+
async executeReactive(input, output) {
|
|
402
|
+
// ✓ Quick preview (< 1ms)
|
|
403
|
+
return { preview: this.quickPreview(input) };
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
async execute(input) {
|
|
407
|
+
// Heavy work belongs here
|
|
408
|
+
const result = await this.trainNeuralNetwork(input);
|
|
409
|
+
return { result };
|
|
410
|
+
}
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
---
|
|
414
|
+
|
|
415
|
+
## Summary
|
|
416
|
+
|
|
417
|
+
| Aspect | `run()` | `runReactive()` |
|
|
418
|
+
| -------------------- | ----------------- | ------------------- |
|
|
419
|
+
| **Purpose** | Full execution | UI previews |
|
|
420
|
+
| **Method called** | `execute()` | `executeReactive()` |
|
|
421
|
+
| **Final status** | COMPLETED | Unchanged |
|
|
422
|
+
| **Output** | Locked/cached | Temporary |
|
|
423
|
+
| **Dataflow updates** | Always | Only PENDING tasks |
|
|
424
|
+
| **Performance** | Can be slow | Should be < 1ms |
|
|
425
|
+
| **User edits** | Before run starts | Before run starts |
|
|
426
|
+
|
|
427
|
+
### Key Takeaways
|
|
428
|
+
|
|
429
|
+
1. Users only edit inputs on PENDING tasks
|
|
430
|
+
2. Once `run()` completes, the task is COMPLETED and immutable
|
|
431
|
+
3. `runReactive()` propagates lightweight updates through PENDING tasks
|
|
432
|
+
4. COMPLETED tasks return cached results in reactive mode
|
|
433
|
+
5. Root tasks in subgraphs receive input from the parent GraphAsTask
|
package/dist/task/BatchTask.d.ts
DELETED
|
@@ -1,154 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @license
|
|
3
|
-
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
import { type DataPortSchema } from "@workglow/util";
|
|
7
|
-
import { IteratorTask, IteratorTaskConfig } from "./IteratorTask";
|
|
8
|
-
import type { TaskInput, TaskOutput, TaskTypeName } from "./TaskTypes";
|
|
9
|
-
/**
|
|
10
|
-
* Configuration for BatchTask.
|
|
11
|
-
*/
|
|
12
|
-
export interface BatchTaskConfig extends IteratorTaskConfig {
|
|
13
|
-
/**
|
|
14
|
-
* Number of items per batch.
|
|
15
|
-
* @default 10
|
|
16
|
-
*/
|
|
17
|
-
readonly batchSize?: number;
|
|
18
|
-
/**
|
|
19
|
-
* Whether to flatten results from all batches into a single array.
|
|
20
|
-
* When false, results are grouped by batch.
|
|
21
|
-
* @default true
|
|
22
|
-
*/
|
|
23
|
-
readonly flattenResults?: boolean;
|
|
24
|
-
/**
|
|
25
|
-
* Whether to execute batches in parallel or sequentially.
|
|
26
|
-
* @default "sequential"
|
|
27
|
-
*/
|
|
28
|
-
readonly batchExecutionMode?: "parallel" | "sequential";
|
|
29
|
-
}
|
|
30
|
-
/**
|
|
31
|
-
* BatchTask processes an array in configurable chunks/batches.
|
|
32
|
-
*
|
|
33
|
-
* This task is useful for:
|
|
34
|
-
* - Rate-limited API calls that accept multiple items
|
|
35
|
-
* - Memory-constrained processing
|
|
36
|
-
* - Progress tracking at batch granularity
|
|
37
|
-
*
|
|
38
|
-
* ## Features
|
|
39
|
-
*
|
|
40
|
-
* - Groups array into chunks of batchSize
|
|
41
|
-
* - Runs inner workflow per batch (receives array of items)
|
|
42
|
-
* - Configurable batch and within-batch execution
|
|
43
|
-
* - Optional result flattening
|
|
44
|
-
*
|
|
45
|
-
* ## Usage
|
|
46
|
-
*
|
|
47
|
-
* ```typescript
|
|
48
|
-
* // Process in batches of 10
|
|
49
|
-
* workflow
|
|
50
|
-
* .input({ documents: [...100 docs...] })
|
|
51
|
-
* .batch({ batchSize: 10 })
|
|
52
|
-
* .bulkEmbed()
|
|
53
|
-
* .bulkStore()
|
|
54
|
-
* .endBatch()
|
|
55
|
-
*
|
|
56
|
-
* // Sequential batches for rate limiting
|
|
57
|
-
* workflow
|
|
58
|
-
* .batch({ batchSize: 5, batchExecutionMode: "sequential" })
|
|
59
|
-
* .apiCall()
|
|
60
|
-
* .endBatch()
|
|
61
|
-
* ```
|
|
62
|
-
*
|
|
63
|
-
* @template Input - The input type containing the array to batch
|
|
64
|
-
* @template Output - The output type (collected batch results)
|
|
65
|
-
* @template Config - The configuration type
|
|
66
|
-
*/
|
|
67
|
-
export declare class BatchTask<Input extends TaskInput = TaskInput, Output extends TaskOutput = TaskOutput, Config extends BatchTaskConfig = BatchTaskConfig> extends IteratorTask<Input, Output, Config> {
|
|
68
|
-
static type: TaskTypeName;
|
|
69
|
-
static category: string;
|
|
70
|
-
static title: string;
|
|
71
|
-
static description: string;
|
|
72
|
-
/**
|
|
73
|
-
* BatchTask always uses PROPERTY_ARRAY merge strategy.
|
|
74
|
-
*/
|
|
75
|
-
static readonly compoundMerge: "PROPERTY_ARRAY";
|
|
76
|
-
/**
|
|
77
|
-
* Static input schema for BatchTask.
|
|
78
|
-
*/
|
|
79
|
-
static inputSchema(): DataPortSchema;
|
|
80
|
-
/**
|
|
81
|
-
* Static output schema for BatchTask.
|
|
82
|
-
*/
|
|
83
|
-
static outputSchema(): DataPortSchema;
|
|
84
|
-
/**
|
|
85
|
-
* Gets the batch size.
|
|
86
|
-
*/
|
|
87
|
-
get batchSize(): number;
|
|
88
|
-
/**
|
|
89
|
-
* Whether to flatten results from all batches.
|
|
90
|
-
*/
|
|
91
|
-
get flattenResults(): boolean;
|
|
92
|
-
/**
|
|
93
|
-
* Batch execution mode.
|
|
94
|
-
*/
|
|
95
|
-
get batchExecutionMode(): "parallel" | "sequential";
|
|
96
|
-
/**
|
|
97
|
-
* Override to group items into batches instead of individual items.
|
|
98
|
-
*/
|
|
99
|
-
protected getIterableItems(input: Input): unknown[];
|
|
100
|
-
/**
|
|
101
|
-
* Groups items into batches of batchSize.
|
|
102
|
-
*/
|
|
103
|
-
protected groupIntoBatches(items: unknown[]): unknown[][];
|
|
104
|
-
/**
|
|
105
|
-
* Creates iteration tasks for batches.
|
|
106
|
-
* Each batch receives the array of items for that batch.
|
|
107
|
-
*/
|
|
108
|
-
protected createIterationTasks(batches: unknown[]): void;
|
|
109
|
-
/**
|
|
110
|
-
* Returns the empty result for BatchTask.
|
|
111
|
-
*/
|
|
112
|
-
protected getEmptyResult(): Output;
|
|
113
|
-
/**
|
|
114
|
-
* Output schema for BatchTask.
|
|
115
|
-
* Similar to MapTask - wraps inner outputs in arrays.
|
|
116
|
-
*/
|
|
117
|
-
outputSchema(): DataPortSchema;
|
|
118
|
-
/**
|
|
119
|
-
* Collects and optionally flattens results from all batches.
|
|
120
|
-
*/
|
|
121
|
-
protected collectResults(results: TaskOutput[]): Output;
|
|
122
|
-
/**
|
|
123
|
-
* Regenerates the graph for batch execution.
|
|
124
|
-
*/
|
|
125
|
-
regenerateGraph(): void;
|
|
126
|
-
}
|
|
127
|
-
declare module "../task-graph/Workflow" {
|
|
128
|
-
interface Workflow {
|
|
129
|
-
/**
|
|
130
|
-
* Starts a batch loop that processes arrays in chunks.
|
|
131
|
-
* Use .endBatch() to close the loop and return to the parent workflow.
|
|
132
|
-
*
|
|
133
|
-
* @param config - Configuration for the batch loop
|
|
134
|
-
* @returns A Workflow in loop builder mode for defining the batch processing
|
|
135
|
-
*
|
|
136
|
-
* @example
|
|
137
|
-
* ```typescript
|
|
138
|
-
* workflow
|
|
139
|
-
* .batch({ batchSize: 10 })
|
|
140
|
-
* .bulkProcess()
|
|
141
|
-
* .endBatch()
|
|
142
|
-
* ```
|
|
143
|
-
*/
|
|
144
|
-
batch: CreateLoopWorkflow<TaskInput, TaskOutput, BatchTaskConfig>;
|
|
145
|
-
/**
|
|
146
|
-
* Ends the batch loop and returns to the parent workflow.
|
|
147
|
-
* Only callable on workflows in loop builder mode.
|
|
148
|
-
*
|
|
149
|
-
* @returns The parent workflow
|
|
150
|
-
*/
|
|
151
|
-
endBatch(): Workflow;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
//# sourceMappingURL=BatchTask.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"BatchTask.d.ts","sourceRoot":"","sources":["../../src/task/BatchTask.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAQrD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD;;;OAGG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,UAAU,GAAG,YAAY,CAAC;CACzD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,SAAS,CACpB,KAAK,SAAS,SAAS,GAAG,SAAS,EACnC,MAAM,SAAS,UAAU,GAAG,UAAU,EACtC,MAAM,SAAS,eAAe,GAAG,eAAe,CAChD,SAAQ,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC;IAC3C,OAAc,IAAI,EAAE,YAAY,CAAe;IAC/C,OAAc,QAAQ,EAAE,MAAM,CAAkB;IAChD,OAAc,KAAK,EAAE,MAAM,CAAW;IACtC,OAAc,WAAW,EAAE,MAAM,CAAgD;IAEjF;;OAEG;IACH,gBAAuB,aAAa,mBAAkB;IAEtD;;OAEG;WACW,WAAW,IAAI,cAAc;IAQ3C;;OAEG;WACW,YAAY,IAAI,cAAc;IAQ5C;;OAEG;IACH,IAAoB,SAAS,IAAI,MAAM,CAEtC;IAED;;OAEG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAED;;OAEG;IACH,IAAW,kBAAkB,IAAI,UAAU,GAAG,YAAY,CAEzD;IAED;;OAEG;cACgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,EAAE;IAK5D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,EAAE;IAWzD;;;OAGG;cACgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI;IA0BjE;;OAEG;cACgB,cAAc,IAAI,MAAM;IAc3C;;;OAGG;IACa,YAAY,IAAI,cAAc;IAQ9C;;OAEG;cACgB,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM;IAqBhE;;OAEG;IACa,eAAe,IAAI,IAAI;CAqBxC;AAMD,OAAO,QAAQ,wBAAwB,CAAC;IACtC,UAAU,QAAQ;QAChB;;;;;;;;;;;;;;WAcG;QAEH,KAAK,EAAE,kBAAkB,CAAC,SAAS,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;QAElE;;;;;WAKG;QACH,QAAQ,IAAI,QAAQ,CAAC;KACtB;CACF"}
|
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @license
|
|
3
|
-
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
import type { DataPortSchema } from "@workglow/util";
|
|
7
|
-
import { IteratorTask, IteratorTaskConfig } from "./IteratorTask";
|
|
8
|
-
import type { TaskInput, TaskOutput, TaskTypeName } from "./TaskTypes";
|
|
9
|
-
/**
|
|
10
|
-
* Configuration for ForEachTask.
|
|
11
|
-
*/
|
|
12
|
-
export interface ForEachTaskConfig extends IteratorTaskConfig {
|
|
13
|
-
/**
|
|
14
|
-
* Whether to collect and return results from each iteration.
|
|
15
|
-
* When false (default), ForEachTask is optimized for side effects.
|
|
16
|
-
* When true, results are collected but not transformed.
|
|
17
|
-
* @default false
|
|
18
|
-
*/
|
|
19
|
-
readonly shouldCollectResults?: boolean;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* ForEachTask iterates over an array and runs a workflow for each element.
|
|
23
|
-
*
|
|
24
|
-
* This task is optimized for side-effect operations where the primary goal
|
|
25
|
-
* is to process each item rather than collect transformed results.
|
|
26
|
-
* For transformations that collect results, use MapTask instead.
|
|
27
|
-
*
|
|
28
|
-
* ## Features
|
|
29
|
-
*
|
|
30
|
-
* - Iterates over array input
|
|
31
|
-
* - Runs inner workflow for each element
|
|
32
|
-
* - Configurable execution modes (parallel, sequential, etc.)
|
|
33
|
-
* - Optimized for side effects (default: doesn't collect results)
|
|
34
|
-
*
|
|
35
|
-
* ## Usage
|
|
36
|
-
*
|
|
37
|
-
* ```typescript
|
|
38
|
-
* // Using Workflow API
|
|
39
|
-
* workflow
|
|
40
|
-
* .input({ items: ["a", "b", "c"] })
|
|
41
|
-
* .forEach()
|
|
42
|
-
* .processItem()
|
|
43
|
-
* .saveToDatabase()
|
|
44
|
-
* .endForEach()
|
|
45
|
-
*
|
|
46
|
-
* // With sequential execution
|
|
47
|
-
* workflow
|
|
48
|
-
* .forEach({ executionMode: "sequential" })
|
|
49
|
-
* .processItem()
|
|
50
|
-
* .endForEach()
|
|
51
|
-
* ```
|
|
52
|
-
*
|
|
53
|
-
* @template Input - The input type containing the array to iterate
|
|
54
|
-
* @template Output - The output type (typically empty for side-effect operations)
|
|
55
|
-
* @template Config - The configuration type
|
|
56
|
-
*/
|
|
57
|
-
export declare class ForEachTask<Input extends TaskInput = TaskInput, Output extends TaskOutput = TaskOutput, Config extends ForEachTaskConfig = ForEachTaskConfig> extends IteratorTask<Input, Output, Config> {
|
|
58
|
-
static type: TaskTypeName;
|
|
59
|
-
static category: string;
|
|
60
|
-
static title: string;
|
|
61
|
-
static description: string;
|
|
62
|
-
/**
|
|
63
|
-
* Static input schema for ForEachTask.
|
|
64
|
-
* Accepts any object with at least one array property.
|
|
65
|
-
*/
|
|
66
|
-
static inputSchema(): DataPortSchema;
|
|
67
|
-
/**
|
|
68
|
-
* Static output schema for ForEachTask.
|
|
69
|
-
* By default, returns an empty object (side-effect focused).
|
|
70
|
-
*/
|
|
71
|
-
static outputSchema(): DataPortSchema;
|
|
72
|
-
/**
|
|
73
|
-
* Whether to collect results from iterations.
|
|
74
|
-
*/
|
|
75
|
-
get shouldCollectResults(): boolean;
|
|
76
|
-
/**
|
|
77
|
-
* Returns the empty result for ForEachTask.
|
|
78
|
-
* Indicates completion with zero items processed.
|
|
79
|
-
*/
|
|
80
|
-
protected getEmptyResult(): Output;
|
|
81
|
-
/**
|
|
82
|
-
* Output schema for ForEachTask instance.
|
|
83
|
-
* If shouldCollectResults is enabled, wraps inner output in arrays.
|
|
84
|
-
* Otherwise, returns the simple completion status schema.
|
|
85
|
-
*/
|
|
86
|
-
outputSchema(): DataPortSchema;
|
|
87
|
-
/**
|
|
88
|
-
* Collects results from all iterations.
|
|
89
|
-
* For ForEachTask, this primarily returns completion status.
|
|
90
|
-
*/
|
|
91
|
-
protected collectResults(results: TaskOutput[]): Output;
|
|
92
|
-
}
|
|
93
|
-
declare module "../task-graph/Workflow" {
|
|
94
|
-
interface Workflow {
|
|
95
|
-
/**
|
|
96
|
-
* Starts a forEach loop that iterates over an array.
|
|
97
|
-
* Use .endForEach() to close the loop and return to the parent workflow.
|
|
98
|
-
*
|
|
99
|
-
* @param config - Configuration for the forEach loop
|
|
100
|
-
* @returns A Workflow in loop builder mode for defining the loop body
|
|
101
|
-
*
|
|
102
|
-
* @example
|
|
103
|
-
* ```typescript
|
|
104
|
-
* workflow
|
|
105
|
-
* .forEach({ executionMode: "sequential" })
|
|
106
|
-
* .processItem()
|
|
107
|
-
* .endForEach()
|
|
108
|
-
* ```
|
|
109
|
-
*/
|
|
110
|
-
forEach: CreateLoopWorkflow<TaskInput, TaskOutput, ForEachTaskConfig>;
|
|
111
|
-
/**
|
|
112
|
-
* Ends the forEach loop and returns to the parent workflow.
|
|
113
|
-
* Only callable on workflows in loop builder mode.
|
|
114
|
-
*
|
|
115
|
-
* @returns The parent workflow
|
|
116
|
-
*/
|
|
117
|
-
endForEach(): Workflow;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
//# sourceMappingURL=ForEachTask.d.ts.map
|