@marktoflow/core 2.0.2 → 2.0.4-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/README.md +69 -6
  2. package/dist/built-in-operations.d.ts +2 -136
  3. package/dist/built-in-operations.d.ts.map +1 -1
  4. package/dist/built-in-operations.js +11 -743
  5. package/dist/built-in-operations.js.map +1 -1
  6. package/dist/engine/conditions.d.ts +29 -0
  7. package/dist/engine/conditions.d.ts.map +1 -0
  8. package/dist/engine/conditions.js +109 -0
  9. package/dist/engine/conditions.js.map +1 -0
  10. package/dist/engine/control-flow.d.ts +35 -0
  11. package/dist/engine/control-flow.d.ts.map +1 -0
  12. package/dist/engine/control-flow.js +653 -0
  13. package/dist/engine/control-flow.js.map +1 -0
  14. package/dist/engine/index.d.ts +12 -0
  15. package/dist/engine/index.d.ts.map +1 -0
  16. package/dist/engine/index.js +11 -0
  17. package/dist/engine/index.js.map +1 -0
  18. package/dist/engine/retry.d.ts +35 -0
  19. package/dist/engine/retry.d.ts.map +1 -0
  20. package/dist/engine/retry.js +86 -0
  21. package/dist/engine/retry.js.map +1 -0
  22. package/dist/engine/subworkflow.d.ts +31 -0
  23. package/dist/engine/subworkflow.d.ts.map +1 -0
  24. package/dist/engine/subworkflow.js +240 -0
  25. package/dist/engine/subworkflow.js.map +1 -0
  26. package/dist/engine/types.d.ts +55 -0
  27. package/dist/engine/types.d.ts.map +1 -0
  28. package/dist/engine/types.js +5 -0
  29. package/dist/{secrets → engine}/types.js.map +1 -1
  30. package/dist/engine/variable-resolution.d.ts +29 -0
  31. package/dist/engine/variable-resolution.d.ts.map +1 -0
  32. package/dist/engine/variable-resolution.js +130 -0
  33. package/dist/engine/variable-resolution.js.map +1 -0
  34. package/dist/engine.d.ts +24 -211
  35. package/dist/engine.d.ts.map +1 -1
  36. package/dist/engine.js +218 -1363
  37. package/dist/engine.js.map +1 -1
  38. package/dist/event-operations.d.ts +59 -0
  39. package/dist/event-operations.d.ts.map +1 -0
  40. package/dist/event-operations.js +99 -0
  41. package/dist/event-operations.js.map +1 -0
  42. package/dist/event-source.d.ts +195 -0
  43. package/dist/event-source.d.ts.map +1 -0
  44. package/dist/event-source.js +757 -0
  45. package/dist/event-source.js.map +1 -0
  46. package/dist/file-operations.js +1 -1
  47. package/dist/file-operations.js.map +1 -1
  48. package/dist/filters/array.d.ts +9 -0
  49. package/dist/filters/array.d.ts.map +1 -0
  50. package/dist/filters/array.js +41 -0
  51. package/dist/filters/array.js.map +1 -0
  52. package/dist/filters/date.d.ts +9 -0
  53. package/dist/filters/date.d.ts.map +1 -0
  54. package/dist/filters/date.js +51 -0
  55. package/dist/filters/date.js.map +1 -0
  56. package/dist/filters/index.d.ts +13 -0
  57. package/dist/filters/index.d.ts.map +1 -0
  58. package/dist/filters/index.js +13 -0
  59. package/dist/filters/index.js.map +1 -0
  60. package/dist/filters/json.d.ts +6 -0
  61. package/dist/filters/json.d.ts.map +1 -0
  62. package/dist/filters/json.js +15 -0
  63. package/dist/filters/json.js.map +1 -0
  64. package/dist/filters/logic.d.ts +8 -0
  65. package/dist/filters/logic.d.ts.map +1 -0
  66. package/dist/filters/logic.js +28 -0
  67. package/dist/filters/logic.js.map +1 -0
  68. package/dist/filters/math.d.ts +13 -0
  69. package/dist/filters/math.d.ts.map +1 -0
  70. package/dist/filters/math.js +39 -0
  71. package/dist/filters/math.js.map +1 -0
  72. package/dist/filters/object.d.ts +11 -0
  73. package/dist/filters/object.d.ts.map +1 -0
  74. package/dist/filters/object.js +64 -0
  75. package/dist/filters/object.js.map +1 -0
  76. package/dist/filters/regex.d.ts +7 -0
  77. package/dist/filters/regex.d.ts.map +1 -0
  78. package/dist/filters/regex.js +38 -0
  79. package/dist/filters/regex.js.map +1 -0
  80. package/dist/filters/string.d.ts +11 -0
  81. package/dist/filters/string.d.ts.map +1 -0
  82. package/dist/filters/string.js +35 -0
  83. package/dist/filters/string.js.map +1 -0
  84. package/dist/filters/type-checks.d.ts +10 -0
  85. package/dist/filters/type-checks.d.ts.map +1 -0
  86. package/dist/filters/type-checks.js +30 -0
  87. package/dist/filters/type-checks.js.map +1 -0
  88. package/dist/index.d.ts +7 -1
  89. package/dist/index.d.ts.map +1 -1
  90. package/dist/index.js +11 -1
  91. package/dist/index.js.map +1 -1
  92. package/dist/models.d.ts +87 -0
  93. package/dist/models.d.ts.map +1 -1
  94. package/dist/models.js +28 -0
  95. package/dist/models.js.map +1 -1
  96. package/dist/nunjucks-filters.d.ts +2 -261
  97. package/dist/nunjucks-filters.d.ts.map +1 -1
  98. package/dist/nunjucks-filters.js +24 -582
  99. package/dist/nunjucks-filters.js.map +1 -1
  100. package/dist/operations/compress.d.ts +6 -0
  101. package/dist/operations/compress.d.ts.map +1 -0
  102. package/dist/operations/compress.js +36 -0
  103. package/dist/operations/compress.js.map +1 -0
  104. package/dist/operations/crypto.d.ts +5 -0
  105. package/dist/operations/crypto.d.ts.map +1 -0
  106. package/dist/operations/crypto.js +61 -0
  107. package/dist/operations/crypto.js.map +1 -0
  108. package/dist/operations/data-ops.d.ts +10 -0
  109. package/dist/operations/data-ops.d.ts.map +1 -0
  110. package/dist/operations/data-ops.js +124 -0
  111. package/dist/operations/data-ops.js.map +1 -0
  112. package/dist/operations/datetime.d.ts +5 -0
  113. package/dist/operations/datetime.d.ts.map +1 -0
  114. package/dist/operations/datetime.js +86 -0
  115. package/dist/operations/datetime.js.map +1 -0
  116. package/dist/operations/extract.d.ts +23 -0
  117. package/dist/operations/extract.d.ts.map +1 -0
  118. package/dist/operations/extract.js +31 -0
  119. package/dist/operations/extract.js.map +1 -0
  120. package/dist/operations/format.d.ts +14 -0
  121. package/dist/operations/format.d.ts.map +1 -0
  122. package/dist/operations/format.js +84 -0
  123. package/dist/operations/format.js.map +1 -0
  124. package/dist/operations/index.d.ts +13 -0
  125. package/dist/operations/index.d.ts.map +1 -0
  126. package/dist/operations/index.js +13 -0
  127. package/dist/operations/index.js.map +1 -0
  128. package/dist/operations/parse.d.ts +5 -0
  129. package/dist/operations/parse.d.ts.map +1 -0
  130. package/dist/operations/parse.js +59 -0
  131. package/dist/operations/parse.js.map +1 -0
  132. package/dist/operations/set.d.ts +21 -0
  133. package/dist/operations/set.d.ts.map +1 -0
  134. package/dist/operations/set.js +25 -0
  135. package/dist/operations/set.js.map +1 -0
  136. package/dist/operations/transform.d.ts +15 -0
  137. package/dist/operations/transform.d.ts.map +1 -0
  138. package/dist/operations/transform.js +110 -0
  139. package/dist/operations/transform.js.map +1 -0
  140. package/dist/parallel.d.ts +114 -0
  141. package/dist/parallel.d.ts.map +1 -0
  142. package/dist/parallel.js +329 -0
  143. package/dist/parallel.js.map +1 -0
  144. package/dist/parser.d.ts.map +1 -1
  145. package/dist/parser.js +23 -4
  146. package/dist/parser.js.map +1 -1
  147. package/dist/permissions.d.ts.map +1 -1
  148. package/dist/permissions.js +17 -2
  149. package/dist/permissions.js.map +1 -1
  150. package/dist/routing.js +2 -2
  151. package/dist/routing.js.map +1 -1
  152. package/dist/sdk-registry.d.ts.map +1 -1
  153. package/dist/sdk-registry.js +9 -3
  154. package/dist/sdk-registry.js.map +1 -1
  155. package/dist/template-engine.d.ts.map +1 -1
  156. package/dist/template-engine.js +10 -15
  157. package/dist/template-engine.js.map +1 -1
  158. package/dist/utils/duration.d.ts +23 -0
  159. package/dist/utils/duration.d.ts.map +1 -0
  160. package/dist/utils/duration.js +41 -0
  161. package/dist/utils/duration.js.map +1 -0
  162. package/dist/utils/errors.d.ts +64 -0
  163. package/dist/utils/errors.d.ts.map +1 -0
  164. package/dist/utils/errors.js +188 -0
  165. package/dist/utils/errors.js.map +1 -0
  166. package/dist/utils/index.d.ts +3 -0
  167. package/dist/utils/index.d.ts.map +1 -0
  168. package/dist/utils/index.js +3 -0
  169. package/dist/utils/index.js.map +1 -0
  170. package/dist/{templates.d.ts → workflow-templates.d.ts} +1 -1
  171. package/dist/workflow-templates.d.ts.map +1 -0
  172. package/dist/{templates.js → workflow-templates.js} +1 -1
  173. package/dist/workflow-templates.js.map +1 -0
  174. package/package.json +34 -7
  175. package/dist/secrets/index.d.ts +0 -12
  176. package/dist/secrets/index.d.ts.map +0 -1
  177. package/dist/secrets/index.js +0 -11
  178. package/dist/secrets/index.js.map +0 -1
  179. package/dist/secrets/providers/aws.d.ts +0 -32
  180. package/dist/secrets/providers/aws.d.ts.map +0 -1
  181. package/dist/secrets/providers/aws.js +0 -118
  182. package/dist/secrets/providers/aws.js.map +0 -1
  183. package/dist/secrets/providers/azure.d.ts +0 -40
  184. package/dist/secrets/providers/azure.d.ts.map +0 -1
  185. package/dist/secrets/providers/azure.js +0 -170
  186. package/dist/secrets/providers/azure.js.map +0 -1
  187. package/dist/secrets/providers/env.d.ts +0 -26
  188. package/dist/secrets/providers/env.d.ts.map +0 -1
  189. package/dist/secrets/providers/env.js +0 -59
  190. package/dist/secrets/providers/env.js.map +0 -1
  191. package/dist/secrets/providers/vault.d.ts +0 -39
  192. package/dist/secrets/providers/vault.d.ts.map +0 -1
  193. package/dist/secrets/providers/vault.js +0 -180
  194. package/dist/secrets/providers/vault.js.map +0 -1
  195. package/dist/secrets/secret-manager.d.ts +0 -72
  196. package/dist/secrets/secret-manager.d.ts.map +0 -1
  197. package/dist/secrets/secret-manager.js +0 -226
  198. package/dist/secrets/secret-manager.js.map +0 -1
  199. package/dist/secrets/types.d.ts +0 -105
  200. package/dist/secrets/types.d.ts.map +0 -1
  201. package/dist/secrets/types.js +0 -8
  202. package/dist/templates.d.ts.map +0 -1
  203. package/dist/templates.js.map +0 -1
package/dist/engine.js CHANGED
@@ -1,279 +1,35 @@
1
1
  /**
2
2
  * Workflow Execution Engine for marktoflow v2.0
3
3
  *
4
- * Executes workflow steps with retry logic, variable resolution,
5
- * and SDK invocation.
4
+ * Thin orchestrator that delegates to focused modules:
5
+ * - engine/control-flow.ts — if, switch, for-each, while, map, filter, reduce, parallel, try, script, wait, merge
6
+ * - engine/retry.ts — RetryPolicy, CircuitBreaker
7
+ * - engine/conditions.ts — condition evaluation
8
+ * - engine/variable-resolution.ts — template and variable resolution
9
+ * - engine/subworkflow.ts — sub-workflow and sub-agent execution
10
+ * - engine/types.ts — shared type definitions
11
+ * - utils/duration.ts — duration parsing
12
+ * - utils/errors.ts — error conversion utilities
6
13
  */
7
14
  import { StepStatus, WorkflowStatus, createExecutionContext, createStepResult, isActionStep, isSubWorkflowStep, isIfStep, isSwitchStep, isForEachStep, isWhileStep, isMapStep, isFilterStep, isReduceStep, isParallelStep, isTryStep, isScriptStep, isWaitStep, isMergeStep, } from './models.js';
8
15
  import { mergePermissions, toSecurityPolicy, } from './permissions.js';
9
16
  import { loadPromptFile, resolvePromptTemplate, validatePromptInputs, } from './prompt-loader.js';
10
17
  import { DEFAULT_FAILOVER_CONFIG, AgentHealthTracker, FailoverReason, } from './failover.js';
11
18
  import { parseFile } from './parser.js';
12
- import { resolve, dirname } from 'node:path';
19
+ import { resolve } from 'node:path';
20
+ import crypto from 'node:crypto';
13
21
  import { executeBuiltInOperation, isBuiltInOperation } from './built-in-operations.js';
14
- import { renderTemplate } from './template-engine.js';
15
- import { executeScriptAsync } from './script-executor.js';
16
- // ============================================================================
17
- // Helper Functions
18
- // ============================================================================
19
- /**
20
- * Parse a duration string like "2h", "30m", "5s", "100ms" into milliseconds.
21
- */
22
- function parseDuration(duration) {
23
- const match = duration.match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$/i);
24
- if (!match) {
25
- const asNum = Number(duration);
26
- if (!isNaN(asNum))
27
- return asNum; // Treat bare numbers as milliseconds
28
- throw new Error(`Invalid duration format: "${duration}". Use format like "2h", "30m", "5s", "100ms"`);
29
- }
30
- const value = parseFloat(match[1]);
31
- const unit = match[2].toLowerCase();
32
- switch (unit) {
33
- case 'ms': return value;
34
- case 's': return value * 1000;
35
- case 'm': return value * 60 * 1000;
36
- case 'h': return value * 60 * 60 * 1000;
37
- case 'd': return value * 24 * 60 * 60 * 1000;
38
- default: return value;
39
- }
40
- }
41
- /**
42
- * Convert error to string for display/logging
43
- */
44
- function errorToString(error) {
45
- if (!error)
46
- return 'Unknown error';
47
- if (typeof error === 'string')
48
- return error;
49
- if (error instanceof Error)
50
- return error.message;
51
- try {
52
- return JSON.stringify(error);
53
- }
54
- catch {
55
- return String(error);
56
- }
57
- }
58
- /**
59
- * Get a field value from an object by key path.
60
- */
61
- function getField(item, field) {
62
- if (!item || typeof item !== 'object')
63
- return undefined;
64
- const parts = field.split('.');
65
- let current = item;
66
- for (const part of parts) {
67
- if (!current || typeof current !== 'object')
68
- return undefined;
69
- current = current[part];
70
- }
71
- return current;
72
- }
73
- // ============================================================================
74
- // Retry Policy
75
- // ============================================================================
76
- export class RetryPolicy {
77
- maxRetries;
78
- baseDelay;
79
- maxDelay;
80
- exponentialBase;
81
- jitter;
82
- constructor(maxRetries = 3, baseDelay = 1000, maxDelay = 30000, exponentialBase = 2, jitter = 0.1) {
83
- this.maxRetries = maxRetries;
84
- this.baseDelay = baseDelay;
85
- this.maxDelay = maxDelay;
86
- this.exponentialBase = exponentialBase;
87
- this.jitter = jitter;
88
- }
89
- /**
90
- * Calculate delay for a given retry attempt.
91
- */
92
- getDelay(attempt) {
93
- const exponentialDelay = this.baseDelay * Math.pow(this.exponentialBase, attempt);
94
- const clampedDelay = Math.min(exponentialDelay, this.maxDelay);
95
- // Add jitter
96
- const jitterAmount = clampedDelay * this.jitter * (Math.random() * 2 - 1);
97
- return Math.max(0, clampedDelay + jitterAmount);
98
- }
99
- }
100
- export class CircuitBreaker {
101
- failureThreshold;
102
- recoveryTimeout;
103
- halfOpenMaxCalls;
104
- state = 'CLOSED';
105
- failures = 0;
106
- lastFailureTime = 0;
107
- halfOpenCalls = 0;
108
- constructor(failureThreshold = 5, recoveryTimeout = 30000, halfOpenMaxCalls = 3) {
109
- this.failureThreshold = failureThreshold;
110
- this.recoveryTimeout = recoveryTimeout;
111
- this.halfOpenMaxCalls = halfOpenMaxCalls;
112
- }
113
- canExecute() {
114
- if (this.state === 'CLOSED') {
115
- return true;
116
- }
117
- if (this.state === 'OPEN') {
118
- const timeSinceFailure = Date.now() - this.lastFailureTime;
119
- if (timeSinceFailure >= this.recoveryTimeout) {
120
- this.state = 'HALF_OPEN';
121
- this.halfOpenCalls = 0;
122
- return true;
123
- }
124
- return false;
125
- }
126
- // HALF_OPEN
127
- return this.halfOpenCalls < this.halfOpenMaxCalls;
128
- }
129
- recordSuccess() {
130
- this.failures = 0;
131
- this.state = 'CLOSED';
132
- }
133
- recordFailure() {
134
- this.failures++;
135
- this.lastFailureTime = Date.now();
136
- if (this.state === 'HALF_OPEN') {
137
- this.state = 'OPEN';
138
- }
139
- else if (this.failures >= this.failureThreshold) {
140
- this.state = 'OPEN';
141
- }
142
- }
143
- getState() {
144
- return this.state;
145
- }
146
- reset() {
147
- this.state = 'CLOSED';
148
- this.failures = 0;
149
- this.halfOpenCalls = 0;
150
- }
151
- }
152
- // ============================================================================
153
- // Variable Resolution
154
- // ============================================================================
155
- /**
156
- * Resolve template variables in a value.
157
- * Supports {{variable}}, {{inputs.name}}, and Nunjucks filters.
158
- *
159
- * Uses Nunjucks as the template engine with:
160
- * - Legacy regex operator support (=~, !~, //) converted to filters
161
- * - Custom filters for string, array, object, date operations
162
- * - Jinja2-style control flow ({% for %}, {% if %}, etc.)
163
- */
164
- export function resolveTemplates(value, context) {
165
- if (typeof value === 'string') {
166
- // Build the template context with all available variables
167
- // Spread inputs first, then variables (variables override inputs if same key)
168
- // Also keep inputs accessible via inputs.* for explicit access
169
- const templateContext = {
170
- ...context.inputs, // Spread inputs at root level for direct access ({{ path }})
171
- ...context.variables, // Variables override inputs if same key
172
- inputs: context.inputs, // Also keep inputs accessible as inputs.*
173
- };
174
- // Use the new Nunjucks-based template engine with legacy syntax support
175
- return renderTemplate(value, templateContext);
176
- }
177
- if (Array.isArray(value)) {
178
- return value.map((v) => resolveTemplates(v, context));
179
- }
180
- if (value && typeof value === 'object') {
181
- const result = {};
182
- for (const [k, v] of Object.entries(value)) {
183
- result[k] = resolveTemplates(v, context);
184
- }
185
- return result;
186
- }
187
- return value;
188
- }
189
- /**
190
- * Resolve a variable path from context.
191
- * First checks inputs.*, then variables, then stepMetadata, then direct context properties.
192
- * Exported to allow access from condition evaluation.
193
- */
194
- export function resolveVariablePath(path, context) {
195
- // Handle inputs.* prefix
196
- if (path.startsWith('inputs.')) {
197
- const inputPath = path.slice(7); // Remove 'inputs.'
198
- return getNestedValue(context.inputs, inputPath);
199
- }
200
- // Check variables first (most common case)
201
- const fromVars = getNestedValue(context.variables, path);
202
- if (fromVars !== undefined) {
203
- return fromVars;
204
- }
205
- // Check inputs (for bare variable names like "value" instead of "inputs.value")
206
- const fromInputs = getNestedValue(context.inputs, path);
207
- if (fromInputs !== undefined) {
208
- return fromInputs;
209
- }
210
- // Check step metadata (for status checks like: step_id.status)
211
- const fromStepMeta = getNestedValue(context.stepMetadata, path);
212
- if (fromStepMeta !== undefined) {
213
- return fromStepMeta;
214
- }
215
- // Fall back to direct context access
216
- return getNestedValue(context, path);
217
- }
218
- /**
219
- * Get a nested value from an object using dot notation and array indexing.
220
- * Supports paths like: "user.name", "items[0].name", "data.users[1].email"
221
- */
222
- function getNestedValue(obj, path) {
223
- if (obj === null || obj === undefined) {
224
- return undefined;
225
- }
226
- // Parse path into parts, handling both dot notation and array indexing
227
- // Convert "a.b[0].c[1]" into ["a", "b", "0", "c", "1"]
228
- const parts = [];
229
- let current = '';
230
- for (let i = 0; i < path.length; i++) {
231
- const char = path[i];
232
- if (char === '.') {
233
- if (current) {
234
- parts.push(current);
235
- current = '';
236
- }
237
- }
238
- else if (char === '[') {
239
- if (current) {
240
- parts.push(current);
241
- current = '';
242
- }
243
- }
244
- else if (char === ']') {
245
- if (current) {
246
- parts.push(current);
247
- current = '';
248
- }
249
- }
250
- else {
251
- current += char;
252
- }
253
- }
254
- if (current) {
255
- parts.push(current);
256
- }
257
- // Traverse the object using the parsed parts
258
- let result = obj;
259
- for (const part of parts) {
260
- if (result === null || result === undefined) {
261
- return undefined;
262
- }
263
- // Check if part is a number (array index)
264
- const index = Number(part);
265
- if (!isNaN(index) && Array.isArray(result)) {
266
- result = result[index];
267
- }
268
- else if (typeof result === 'object') {
269
- result = result[part];
270
- }
271
- else {
272
- return undefined;
273
- }
274
- }
275
- return result;
276
- }
22
+ import { executeParallelOperation, isParallelOperation } from './parallel.js';
23
+ // Engine sub-modules
24
+ import { RetryPolicy, CircuitBreaker } from './engine/retry.js';
25
+ import { evaluateConditions } from './engine/conditions.js';
26
+ import { resolveTemplates } from './engine/variable-resolution.js';
27
+ import { executeIfStep, executeSwitchStep, executeForEachStep, executeWhileStep, executeMapStep, executeFilterStep, executeReduceStep, executeParallelStep, executeTryStep, executeScriptStep, executeWaitStep, executeMergeStep, } from './engine/control-flow.js';
28
+ import { executeSubWorkflow, executeSubWorkflowWithAgent, } from './engine/subworkflow.js';
29
+ import { errorToString } from './utils/errors.js';
30
+ // Re-export types and classes for backward compatibility
31
+ export { RetryPolicy, CircuitBreaker } from './engine/retry.js';
32
+ export { resolveTemplates, resolveVariablePath } from './engine/variable-resolution.js';
277
33
  export class WorkflowEngine {
278
34
  config;
279
35
  retryPolicy;
@@ -284,15 +40,15 @@ export class WorkflowEngine {
284
40
  failoverConfig;
285
41
  healthTracker;
286
42
  failoverEvents = [];
287
- workflowPath; // Base path for resolving sub-workflows (public for CLI state tracking)
288
- workflowPermissions; // Workflow-level permissions
289
- promptCache = new Map(); // Cache for loaded prompts
43
+ workflowPath;
44
+ workflowPermissions;
45
+ promptCache = new Map();
290
46
  constructor(config = {}, events = {}, stateStore) {
291
47
  this.config = {
292
- defaultTimeout: config.defaultTimeout ?? 60000,
293
- maxRetries: config.maxRetries ?? 3,
294
- retryBaseDelay: config.retryBaseDelay ?? 1000,
295
- retryMaxDelay: config.retryMaxDelay ?? 30000,
48
+ defaultTimeout: config.defaultTimeout ?? (process.env.MARKTOFLOW_TIMEOUT ? parseInt(process.env.MARKTOFLOW_TIMEOUT, 10) : 60000),
49
+ maxRetries: config.maxRetries ?? (process.env.MARKTOFLOW_MAX_RETRIES ? parseInt(process.env.MARKTOFLOW_MAX_RETRIES, 10) : 3),
50
+ retryBaseDelay: config.retryBaseDelay ?? (process.env.MARKTOFLOW_RETRY_BASE_DELAY ? parseInt(process.env.MARKTOFLOW_RETRY_BASE_DELAY, 10) : 1000),
51
+ retryMaxDelay: config.retryMaxDelay ?? (process.env.MARKTOFLOW_RETRY_MAX_DELAY ? parseInt(process.env.MARKTOFLOW_RETRY_MAX_DELAY, 10) : 30000),
296
52
  defaultAgent: config.defaultAgent,
297
53
  defaultModel: config.defaultModel,
298
54
  };
@@ -303,58 +59,70 @@ export class WorkflowEngine {
303
59
  this.failoverConfig = { ...DEFAULT_FAILOVER_CONFIG, ...(config.failoverConfig ?? {}) };
304
60
  this.healthTracker = config.healthTracker ?? new AgentHealthTracker();
305
61
  }
62
+ // ============================================================================
63
+ // Step Dispatcher
64
+ // ============================================================================
306
65
  /**
307
- * Execute a single step - dispatcher to specialized execution methods.
66
+ * Execute a single step - dispatches to specialized execution methods.
308
67
  */
309
68
  async executeStep(step, context, sdkRegistry, stepExecutor) {
310
69
  // Check conditions first (applies to all step types)
311
- if (step.conditions && !this.evaluateConditions(step.conditions, context)) {
70
+ if (step.conditions && !evaluateConditions(step.conditions, context)) {
312
71
  return createStepResult(step.id, StepStatus.SKIPPED, null, new Date());
313
72
  }
73
+ // Bind the dispatcher for recursive step execution
74
+ const dispatch = (s, c, sr, se) => this.executeStep(s, c, sr, se);
314
75
  // Dispatch to specialized execution method based on step type
315
76
  if (isIfStep(step)) {
316
- return this.executeIfStep(step, context, sdkRegistry, stepExecutor);
77
+ return executeIfStep(step, context, sdkRegistry, stepExecutor, dispatch);
317
78
  }
318
79
  if (isSwitchStep(step)) {
319
- return this.executeSwitchStep(step, context, sdkRegistry, stepExecutor);
80
+ return executeSwitchStep(step, context, sdkRegistry, stepExecutor, dispatch);
320
81
  }
321
82
  if (isForEachStep(step)) {
322
- return this.executeForEachStep(step, context, sdkRegistry, stepExecutor);
83
+ return executeForEachStep(step, context, sdkRegistry, stepExecutor, dispatch);
323
84
  }
324
85
  if (isWhileStep(step)) {
325
- return this.executeWhileStep(step, context, sdkRegistry, stepExecutor);
86
+ return executeWhileStep(step, context, sdkRegistry, stepExecutor, dispatch);
326
87
  }
327
88
  if (isMapStep(step)) {
328
- return this.executeMapStep(step, context, sdkRegistry, stepExecutor);
89
+ return executeMapStep(step, context);
329
90
  }
330
91
  if (isFilterStep(step)) {
331
- return this.executeFilterStep(step, context, sdkRegistry, stepExecutor);
92
+ return executeFilterStep(step, context);
332
93
  }
333
94
  if (isReduceStep(step)) {
334
- return this.executeReduceStep(step, context, sdkRegistry, stepExecutor);
95
+ return executeReduceStep(step, context);
335
96
  }
336
97
  if (isParallelStep(step)) {
337
- return this.executeParallelStep(step, context, sdkRegistry, stepExecutor);
98
+ return executeParallelStep(step, context, sdkRegistry, stepExecutor, dispatch, (c) => this.cloneContext(c), (main, branch, id) => this.mergeContexts(main, branch, id), (promises, limit) => this.executeConcurrentlyWithLimit(promises, limit));
338
99
  }
339
100
  if (isTryStep(step)) {
340
- return this.executeTryStep(step, context, sdkRegistry, stepExecutor);
101
+ return executeTryStep(step, context, sdkRegistry, stepExecutor, dispatch);
341
102
  }
342
103
  if (isScriptStep(step)) {
343
- return this.executeScriptStep(step, context);
104
+ return executeScriptStep(step, context);
344
105
  }
345
106
  if (isWaitStep(step)) {
346
- return this.executeWaitStep(step, context);
107
+ return executeWaitStep(step, context, this.stateStore);
347
108
  }
348
109
  if (isMergeStep(step)) {
349
- return this.executeMergeStep(step, context);
110
+ return executeMergeStep(step, context);
350
111
  }
351
112
  // Default: action or workflow step
352
113
  return this.executeStepWithFailover(step, context, sdkRegistry, stepExecutor);
353
114
  }
115
+ // ============================================================================
116
+ // Workflow Execution
117
+ // ============================================================================
354
118
  /**
355
119
  * Execute a workflow.
356
120
  */
357
121
  async execute(workflow, inputs = {}, sdkRegistry, stepExecutor) {
122
+ // Daemon/event mode: delegate to continuous execution
123
+ if (workflow.mode === 'daemon' || workflow.mode === 'event') {
124
+ return this.executeDaemon(workflow, inputs, sdkRegistry, stepExecutor);
125
+ }
358
126
  const context = createExecutionContext(workflow, inputs);
359
127
  const stepResults = [];
360
128
  const startedAt = new Date();
@@ -389,11 +157,9 @@ export class WorkflowEngine {
389
157
  for (let i = 0; i < workflow.steps.length; i++) {
390
158
  const step = workflow.steps[i];
391
159
  context.currentStepIndex = i;
392
- // Execute step using dispatcher
393
160
  const result = await this.executeStep(step, context, sdkRegistry, stepExecutor);
394
161
  stepResults.push(result);
395
- // Store step metadata (status, error, etc.) in separate field for condition evaluation
396
- // This allows conditions like: step_id.status == 'failed'
162
+ // Store step metadata for condition evaluation
397
163
  context.stepMetadata[step.id] = {
398
164
  status: result.status.toLowerCase(),
399
165
  retryCount: result.retryCount,
@@ -414,7 +180,6 @@ export class WorkflowEngine {
414
180
  }
415
181
  // Handle failure
416
182
  if (result.status === StepStatus.FAILED) {
417
- // Get error action from step if it has error handling
418
183
  let errorAction = 'stop';
419
184
  if ('errorHandling' in step && step.errorHandling?.action) {
420
185
  errorAction = step.errorHandling.action;
@@ -426,7 +191,6 @@ export class WorkflowEngine {
426
191
  this.events.onWorkflowComplete?.(workflow, workflowResult);
427
192
  return workflowResult;
428
193
  }
429
- // 'continue' - keep going
430
194
  if (errorAction === 'rollback') {
431
195
  if (this.rollbackRegistry) {
432
196
  await this.rollbackRegistry.rollbackAllAsync({
@@ -443,7 +207,6 @@ export class WorkflowEngine {
443
207
  }
444
208
  }
445
209
  }
446
- // Determine final status
447
210
  context.status = WorkflowStatus.COMPLETED;
448
211
  }
449
212
  catch (error) {
@@ -470,21 +233,109 @@ export class WorkflowEngine {
470
233
  this.events.onWorkflowComplete?.(workflow, workflowResult);
471
234
  return workflowResult;
472
235
  }
236
+ /**
237
+ * Execute workflow in daemon/continuous mode.
238
+ * Automatically connects to event sources from frontmatter and loops execution.
239
+ */
240
+ async executeDaemon(workflow, inputs, sdkRegistry, stepExecutor) {
241
+ const startedAt = new Date();
242
+ // Store workflow-level permissions and defaults
243
+ this.workflowPermissions = workflow.permissions;
244
+ if (!this.config.defaultAgent && workflow.defaultAgent) {
245
+ this.config.defaultAgent = workflow.defaultAgent;
246
+ }
247
+ if (!this.config.defaultModel && workflow.defaultModel) {
248
+ this.config.defaultModel = workflow.defaultModel;
249
+ }
250
+ // Auto-connect to event sources from frontmatter
251
+ if (workflow.sources && workflow.sources.length > 0) {
252
+ const { getEventSourceManager } = await import('./event-operations.js');
253
+ const manager = getEventSourceManager();
254
+ for (const source of workflow.sources) {
255
+ try {
256
+ // Clean config: remove undefined properties for exactOptionalPropertyTypes
257
+ const config = {
258
+ kind: source.kind,
259
+ id: source.id,
260
+ options: source.options,
261
+ ...(source.filter ? { filter: source.filter } : {}),
262
+ ...(source.reconnect !== undefined ? { reconnect: source.reconnect } : {}),
263
+ ...(source.reconnectDelay !== undefined ? { reconnectDelay: source.reconnectDelay } : {}),
264
+ };
265
+ await manager.add(config);
266
+ }
267
+ catch (error) {
268
+ const errorMsg = `Failed to connect to event source '${source.id}': ${error instanceof Error ? error.message : String(error)}`;
269
+ return {
270
+ workflowId: workflow.metadata.id,
271
+ runId: crypto.randomUUID(),
272
+ status: WorkflowStatus.FAILED,
273
+ stepResults: [],
274
+ output: {},
275
+ error: errorMsg,
276
+ startedAt,
277
+ completedAt: new Date(),
278
+ duration: Date.now() - startedAt.getTime(),
279
+ };
280
+ }
281
+ }
282
+ }
283
+ // Daemon mode: run indefinitely until killed
284
+ // eslint-disable-next-line no-constant-condition
285
+ while (true) {
286
+ const context = createExecutionContext(workflow, inputs);
287
+ const stepResults = [];
288
+ context.status = WorkflowStatus.RUNNING;
289
+ this.events.onWorkflowStart?.(workflow, context);
290
+ try {
291
+ for (let i = 0; i < workflow.steps.length; i++) {
292
+ const step = workflow.steps[i];
293
+ context.currentStepIndex = i;
294
+ const result = await this.executeStep(step, context, sdkRegistry, stepExecutor);
295
+ stepResults.push(result);
296
+ // Store step metadata
297
+ context.stepMetadata[step.id] = {
298
+ status: result.status.toLowerCase(),
299
+ retryCount: result.retryCount,
300
+ ...(result.error ? { error: errorToString(result.error) } : {}),
301
+ };
302
+ // Store output variable
303
+ if (step.outputVariable && result.status === StepStatus.COMPLETED) {
304
+ context.variables[step.outputVariable] = result.output;
305
+ }
306
+ // Handle failure
307
+ if (result.status === StepStatus.FAILED) {
308
+ let errorAction = 'stop';
309
+ if ('errorHandling' in step && step.errorHandling?.action) {
310
+ errorAction = step.errorHandling.action;
311
+ }
312
+ if (errorAction === 'stop') {
313
+ // In daemon mode, log error and restart loop
314
+ const error = errorToString(result.error);
315
+ console.error(`[Daemon] Step ${step.id} failed: ${error}`);
316
+ context.status = WorkflowStatus.FAILED;
317
+ break;
318
+ }
319
+ }
320
+ }
321
+ context.status = WorkflowStatus.COMPLETED;
322
+ }
323
+ catch (error) {
324
+ const errorMsg = error instanceof Error ? error.message : String(error);
325
+ console.error(`[Daemon] Workflow error: ${errorMsg}`);
326
+ context.status = WorkflowStatus.FAILED;
327
+ }
328
+ // Brief pause before next iteration
329
+ await new Promise(resolve => setTimeout(resolve, 100));
330
+ }
331
+ }
473
332
  /**
474
333
  * Resume a paused execution (e.g., after form submission).
475
- *
476
- * @param runId - The execution run ID
477
- * @param stepId - The step ID that was waiting
478
- * @param resumeData - Data from the resume action (e.g., form submission)
479
- * @param sdkRegistry - SDK registry for step execution
480
- * @param stepExecutor - Step executor function
481
- * @returns Workflow result from resumed execution
482
334
  */
483
335
  async resumeExecution(runId, stepId, resumeData, sdkRegistry, stepExecutor) {
484
336
  if (!this.stateStore) {
485
337
  throw new Error('Cannot resume execution: StateStore not configured');
486
338
  }
487
- // Load execution from state store
488
339
  const execution = this.stateStore.getExecution(runId);
489
340
  if (!execution) {
490
341
  throw new Error(`Execution ${runId} not found`);
@@ -492,10 +343,8 @@ export class WorkflowEngine {
492
343
  if (execution.status !== WorkflowStatus.RUNNING) {
493
344
  throw new Error(`Cannot resume execution ${runId}: status is ${execution.status}`);
494
345
  }
495
- // Load workflow
496
346
  const { workflow } = await parseFile(execution.workflowPath);
497
347
  this.workflowPath = execution.workflowPath;
498
- // Find the step that was waiting
499
348
  const stepIndex = workflow.steps.findIndex(s => s.id === stepId);
500
349
  if (stepIndex === -1) {
501
350
  throw new Error(`Step ${stepId} not found in workflow`);
@@ -511,16 +360,13 @@ export class WorkflowEngine {
511
360
  const checkpoint = checkpoints.find(cp => cp.stepIndex === i);
512
361
  if (checkpoint) {
513
362
  const step = workflow.steps[i];
514
- // Recreate step result
515
363
  const result = createStepResult(step.id, checkpoint.status, checkpoint.outputs, checkpoint.startedAt, checkpoint.retryCount, checkpoint.error || undefined);
516
364
  stepResults.push(result);
517
- // Restore step metadata
518
365
  context.stepMetadata[step.id] = {
519
366
  status: checkpoint.status.toLowerCase(),
520
367
  retryCount: checkpoint.retryCount,
521
368
  ...(checkpoint.error ? { error: checkpoint.error } : {}),
522
369
  };
523
- // Restore output variable
524
370
  if (step.outputVariable && checkpoint.status === StepStatus.COMPLETED) {
525
371
  context.variables[step.outputVariable] = checkpoint.outputs;
526
372
  }
@@ -602,273 +448,26 @@ export class WorkflowEngine {
602
448
  }
603
449
  /**
604
450
  * Execute a workflow from a file.
605
- * This method automatically sets the workflow path for resolving sub-workflows.
606
451
  */
607
452
  async executeFile(workflowPath, inputs = {}, sdkRegistry, stepExecutor) {
608
- // Parse the workflow file
609
453
  const { workflow } = await parseFile(workflowPath);
610
- // Set the workflow path for sub-workflow resolution
611
454
  this.workflowPath = resolve(workflowPath);
612
- // Execute the workflow
613
455
  return this.execute(workflow, inputs, sdkRegistry, stepExecutor);
614
456
  }
615
457
  getFailoverHistory() {
616
458
  return [...this.failoverEvents];
617
459
  }
618
- /**
619
- * Execute a sub-workflow.
620
- */
621
- async executeSubWorkflow(step, context, sdkRegistry, stepExecutor) {
622
- if (!isSubWorkflowStep(step)) {
623
- throw new Error(`Step ${step.id} is not a workflow step`);
624
- }
625
- // Resolve the sub-workflow path relative to the parent workflow
626
- const subWorkflowPath = this.workflowPath
627
- ? resolve(dirname(this.workflowPath), step.workflow)
628
- : resolve(step.workflow);
629
- // Parse the sub-workflow
630
- const { workflow: subWorkflow } = await parseFile(subWorkflowPath);
631
- // Resolve inputs for the sub-workflow
632
- const resolvedInputs = resolveTemplates(step.inputs, context);
633
- // Create a new engine instance for the sub-workflow with the same configuration
634
- const subEngineConfig = {
635
- defaultTimeout: this.config.defaultTimeout,
636
- maxRetries: this.config.maxRetries,
637
- retryBaseDelay: this.config.retryBaseDelay,
638
- retryMaxDelay: this.config.retryMaxDelay,
639
- failoverConfig: this.failoverConfig,
640
- healthTracker: this.healthTracker,
641
- };
642
- if (this.rollbackRegistry) {
643
- subEngineConfig.rollbackRegistry = this.rollbackRegistry;
644
- }
645
- const subEngine = new WorkflowEngine(subEngineConfig, this.events, this.stateStore);
646
- // Set the base path for the sub-workflow
647
- subEngine.workflowPath = subWorkflowPath;
648
- // Execute the sub-workflow
649
- const result = await subEngine.execute(subWorkflow, resolvedInputs, sdkRegistry, stepExecutor);
650
- // Check if sub-workflow failed
651
- if (result.status === WorkflowStatus.FAILED) {
652
- throw new Error(result.error || 'Sub-workflow execution failed');
653
- }
654
- // Return the sub-workflow output
655
- return result.output;
656
- }
657
- /**
658
- * Execute a sub-workflow using an AI sub-agent.
659
- * The agent interprets the workflow and executes it autonomously.
660
- */
661
- async executeSubWorkflowWithAgent(step, context, sdkRegistry, stepExecutor) {
662
- // Resolve the sub-workflow path
663
- const subWorkflowPath = this.workflowPath
664
- ? resolve(dirname(this.workflowPath), step.workflow)
665
- : resolve(step.workflow);
666
- // Read the workflow file content
667
- const { readFile } = await import('node:fs/promises');
668
- const workflowContent = await readFile(subWorkflowPath, 'utf-8');
669
- // Resolve inputs for the sub-workflow
670
- const resolvedInputs = resolveTemplates(step.inputs, context);
671
- // Get subagent configuration
672
- const subagentConfig = step.subagentConfig || {};
673
- const model = subagentConfig.model || step.model || this.config.defaultModel;
674
- const maxTurns = subagentConfig.maxTurns || 10;
675
- const systemPrompt = subagentConfig.systemPrompt || this.buildDefaultSubagentSystemPrompt();
676
- const tools = subagentConfig.tools || ['Read', 'Write', 'Bash', 'Glob', 'Grep'];
677
- // Build the prompt for the agent
678
- const agentPrompt = this.buildSubagentPrompt(workflowContent, resolvedInputs, tools);
679
- // Determine the agent action to use
680
- const agentName = step.agent || this.config.defaultAgent || 'agent';
681
- const agentAction = `${agentName}.chat.completions`;
682
- // Build the messages array
683
- const messages = [
684
- { role: 'system', content: systemPrompt },
685
- { role: 'user', content: agentPrompt },
686
- ];
687
- // Create a virtual action step to execute via the agent
688
- const agentStep = {
689
- id: `${step.id}-subagent`,
690
- type: 'action',
691
- action: agentAction,
692
- inputs: {
693
- model,
694
- messages,
695
- max_tokens: 8192,
696
- },
697
- model,
698
- agent: agentName,
699
- };
700
- // Build executor context
701
- const executorContext = this.buildStepExecutorContext(agentStep);
702
- // Execute the agent call
703
- let response;
704
- let turns = 0;
705
- let conversationMessages = [...messages];
706
- let finalOutput = {};
707
- while (turns < maxTurns) {
708
- turns++;
709
- try {
710
- response = await stepExecutor({ ...agentStep, inputs: { ...agentStep.inputs, messages: conversationMessages } }, context, sdkRegistry, executorContext);
711
- // Parse the response
712
- const parsedResponse = this.parseSubagentResponse(response);
713
- if (parsedResponse.completed) {
714
- finalOutput = parsedResponse.output || {};
715
- break;
716
- }
717
- // If agent needs to continue, add its response and continue
718
- if (parsedResponse.message) {
719
- conversationMessages.push({ role: 'assistant', content: parsedResponse.message });
720
- // Agent might request a tool call - for now, we'll prompt it to continue
721
- conversationMessages.push({ role: 'user', content: 'Continue with the workflow execution.' });
722
- }
723
- else {
724
- // No clear continuation, assume completed
725
- finalOutput = parsedResponse.output || {};
726
- break;
727
- }
728
- }
729
- catch (error) {
730
- throw new Error(`Sub-agent execution failed at turn ${turns}: ${error instanceof Error ? error.message : String(error)}`);
731
- }
732
- }
733
- if (turns >= maxTurns) {
734
- throw new Error(`Sub-agent exceeded maximum turns (${maxTurns})`);
735
- }
736
- return finalOutput;
737
- }
738
- /**
739
- * Build the default system prompt for sub-agent execution.
740
- */
741
- buildDefaultSubagentSystemPrompt() {
742
- return `You are an AI agent executing a workflow. Your task is to interpret the workflow definition and execute each step in order.
743
-
744
- For each step:
745
- 1. Understand what the step requires
746
- 2. Execute the action described
747
- 3. Store any outputs as specified
748
-
749
- When you complete all steps, respond with a JSON object containing the workflow outputs.
750
-
751
- Format your final response as:
752
- \`\`\`json
753
- {
754
- "completed": true,
755
- "output": { /* workflow outputs here */ }
756
- }
757
- \`\`\`
758
-
759
- If you encounter an error, respond with:
760
- \`\`\`json
761
- {
762
- "completed": false,
763
- "error": "description of the error"
764
- }
765
- \`\`\``;
766
- }
767
- /**
768
- * Build the prompt for sub-agent workflow execution.
769
- */
770
- buildSubagentPrompt(workflowContent, inputs, tools) {
771
- return `Execute the following workflow:
772
-
773
- ## Workflow Definition
774
- \`\`\`markdown
775
- ${workflowContent}
776
- \`\`\`
777
-
778
- ## Inputs
779
- \`\`\`json
780
- ${JSON.stringify(inputs, null, 2)}
781
- \`\`\`
782
-
783
- ## Available Tools
784
- ${tools.join(', ')}
785
-
786
- Execute the workflow steps in order and return the final outputs as JSON.`;
787
- }
788
- /**
789
- * Parse the sub-agent's response to extract completion status and output.
790
- */
791
- parseSubagentResponse(response) {
792
- // Try to extract content from various response formats
793
- let content;
794
- if (typeof response === 'string') {
795
- content = response;
796
- }
797
- else if (response && typeof response === 'object') {
798
- const resp = response;
799
- // OpenAI-style response
800
- if (resp.choices && Array.isArray(resp.choices)) {
801
- const choice = resp.choices[0];
802
- if (choice.message && typeof choice.message === 'object') {
803
- const message = choice.message;
804
- content = message.content;
805
- }
806
- }
807
- // Anthropic-style response
808
- else if (resp.content && Array.isArray(resp.content)) {
809
- const textBlock = resp.content.find((c) => typeof c === 'object' && c !== null && c.type === 'text');
810
- content = textBlock?.text;
811
- }
812
- // Direct content field
813
- else if (typeof resp.content === 'string') {
814
- content = resp.content;
815
- }
816
- // Direct message field
817
- else if (typeof resp.message === 'string') {
818
- content = resp.message;
819
- }
820
- }
821
- if (!content) {
822
- return { completed: false, message: 'No content in response' };
823
- }
824
- // Try to parse JSON from the response
825
- const jsonMatch = content.match(/```json\n?([\s\S]*?)```/);
826
- if (jsonMatch) {
827
- try {
828
- const parsed = JSON.parse(jsonMatch[1]);
829
- const output = parsed.output;
830
- const error = parsed.error;
831
- return {
832
- completed: parsed.completed === true,
833
- ...(output !== undefined ? { output } : {}),
834
- ...(error !== undefined ? { error } : {}),
835
- };
836
- }
837
- catch (e) {
838
- console.warn('[marktoflow] Failed to parse JSON block in sub-agent response:', e.message);
839
- }
840
- }
841
- // Try to parse raw JSON
842
- try {
843
- const parsed = JSON.parse(content);
844
- if (typeof parsed.completed === 'boolean') {
845
- const output = parsed.output;
846
- const error = parsed.error;
847
- return {
848
- completed: parsed.completed,
849
- ...(output !== undefined ? { output } : {}),
850
- ...(error !== undefined ? { error } : {}),
851
- };
852
- }
853
- }
854
- catch (e) {
855
- console.warn('[marktoflow] Sub-agent response is not valid JSON:', e.message);
856
- }
857
- // Return the content as a message
858
- return { completed: false, message: content };
859
- }
460
+ // ============================================================================
461
+ // Step Execution with Retry & Failover
462
+ // ============================================================================
860
463
  /**
861
464
  * Build the step executor context with effective model/agent/permissions.
862
465
  */
863
466
  buildStepExecutorContext(step) {
864
- // Merge workflow and step permissions
865
467
  const effectivePermissions = mergePermissions(this.workflowPermissions, step.permissions);
866
- // Resolve effective model/agent (step overrides workflow defaults)
867
- const effectiveModel = step.model || this.config.defaultModel;
868
- const effectiveAgent = step.agent || this.config.defaultAgent;
869
468
  return {
870
- model: effectiveModel,
871
- agent: effectiveAgent,
469
+ model: step.model || this.config.defaultModel,
470
+ agent: step.agent || this.config.defaultAgent,
872
471
  permissions: effectivePermissions,
873
472
  securityPolicy: toSecurityPolicy(effectivePermissions),
874
473
  basePath: this.workflowPath,
@@ -881,33 +480,25 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
881
480
  if (!step.prompt) {
882
481
  return step.inputs;
883
482
  }
884
- // Check cache
885
483
  let loadedPrompt = this.promptCache.get(step.prompt);
886
484
  if (!loadedPrompt) {
887
485
  loadedPrompt = await loadPromptFile(step.prompt, this.workflowPath);
888
486
  this.promptCache.set(step.prompt, loadedPrompt);
889
487
  }
890
- // Resolve prompt inputs (from step.promptInputs, with template resolution)
891
488
  const promptInputs = step.promptInputs
892
489
  ? resolveTemplates(step.promptInputs, context)
893
490
  : {};
894
- // Validate prompt inputs
895
491
  const validation = validatePromptInputs(loadedPrompt, promptInputs);
896
492
  if (!validation.valid) {
897
493
  throw new Error(`Invalid prompt inputs: ${validation.errors.join(', ')}`);
898
494
  }
899
- // Resolve the prompt template
900
495
  const resolved = resolvePromptTemplate(loadedPrompt, promptInputs, context);
901
- // Merge resolved prompt content into inputs
902
- // The resolved content typically goes into a 'messages' or 'prompt' field
903
496
  const resolvedInputs = { ...step.inputs };
904
- // If inputs has a 'messages' array with a user message, inject prompt content
905
497
  if (Array.isArray(resolvedInputs.messages)) {
906
498
  resolvedInputs.messages = resolvedInputs.messages.map((msg) => {
907
499
  if (typeof msg === 'object' && msg !== null) {
908
500
  const message = msg;
909
501
  if (message.role === 'user' && typeof message.content === 'string') {
910
- // Replace {{ prompt }} placeholder with resolved content
911
502
  return {
912
503
  ...message,
913
504
  content: message.content.replace(/\{\{\s*prompt\s*\}\}/g, resolved.content),
@@ -918,7 +509,6 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
918
509
  });
919
510
  }
920
511
  else {
921
- // Add resolved prompt as 'promptContent' for the executor to use
922
512
  resolvedInputs.promptContent = resolved.content;
923
513
  }
924
514
  return resolvedInputs;
@@ -929,15 +519,13 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
929
519
  async executeStepWithRetry(step, context, sdkRegistry, stepExecutor) {
930
520
  const startedAt = new Date();
931
521
  let lastError;
932
- // Build executor context with model/agent/permissions
933
522
  const executorContext = this.buildStepExecutorContext(step);
934
523
  // Handle sub-workflow execution
935
524
  if (isSubWorkflowStep(step)) {
936
- // Check if we should use subagent execution
937
525
  if (step.useSubagent) {
938
526
  try {
939
527
  this.events.onStepStart?.(step, context);
940
- const output = await this.executeWithTimeout(() => this.executeSubWorkflowWithAgent(step, context, sdkRegistry, stepExecutor), step.timeout ?? this.config.defaultTimeout);
528
+ const output = await this.executeWithTimeout(() => executeSubWorkflowWithAgent(step, context, sdkRegistry, stepExecutor, this.workflowPath, this.config.defaultModel, this.config.defaultAgent, (s) => this.buildStepExecutorContext(s)), step.timeout ?? this.config.defaultTimeout);
941
529
  const result = createStepResult(step.id, StepStatus.COMPLETED, output, startedAt, 0);
942
530
  this.events.onStepComplete?.(step, result);
943
531
  return result;
@@ -952,25 +540,35 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
952
540
  // Standard sub-workflow execution
953
541
  try {
954
542
  this.events.onStepStart?.(step, context);
955
- const output = await this.executeWithTimeout(() => this.executeSubWorkflow(step, context, sdkRegistry, stepExecutor), step.timeout ?? this.config.defaultTimeout);
543
+ const createSubEngine = (cfg) => {
544
+ const subEngine = new WorkflowEngine(cfg, this.events, this.stateStore);
545
+ return subEngine;
546
+ };
547
+ const output = await this.executeWithTimeout(() => executeSubWorkflow(step, context, sdkRegistry, stepExecutor, this.workflowPath, createSubEngine, {
548
+ defaultTimeout: this.config.defaultTimeout,
549
+ maxRetries: this.config.maxRetries,
550
+ retryBaseDelay: this.config.retryBaseDelay,
551
+ retryMaxDelay: this.config.retryMaxDelay,
552
+ failoverConfig: this.failoverConfig,
553
+ healthTracker: this.healthTracker,
554
+ ...(this.rollbackRegistry ? { rollbackRegistry: this.rollbackRegistry } : {}),
555
+ }), step.timeout ?? this.config.defaultTimeout);
956
556
  const result = createStepResult(step.id, StepStatus.COMPLETED, output, startedAt, 0);
957
557
  this.events.onStepComplete?.(step, result);
958
558
  return result;
959
559
  }
960
560
  catch (error) {
961
561
  lastError = error instanceof Error ? error : new Error(String(error));
962
- const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, lastError // Pass full error object
963
- );
562
+ const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, lastError);
964
563
  this.events.onStepComplete?.(step, result);
965
564
  return result;
966
565
  }
967
566
  }
968
- // Regular action step - ensure action is defined
567
+ // Regular action step
969
568
  if (!isActionStep(step)) {
970
569
  return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Step is neither an action nor a workflow');
971
570
  }
972
571
  const maxRetries = step.errorHandling?.maxRetries ?? this.config.maxRetries;
973
- // Get or create circuit breaker for this step's action
974
572
  const [serviceName] = step.action.split('.');
975
573
  let circuitBreaker = this.circuitBreakers.get(serviceName);
976
574
  if (!circuitBreaker) {
@@ -978,34 +576,32 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
978
576
  this.circuitBreakers.set(serviceName, circuitBreaker);
979
577
  }
980
578
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
981
- // Check circuit breaker
982
579
  if (!circuitBreaker.canExecute()) {
983
580
  return createStepResult(step.id, StepStatus.FAILED, null, startedAt, attempt, `Circuit breaker open for service: ${serviceName}`);
984
581
  }
985
582
  this.events.onStepStart?.(step, context);
986
583
  try {
987
- // Load and resolve external prompt if specified
988
584
  let resolvedInputs;
989
585
  if (step.prompt) {
990
586
  resolvedInputs = await this.loadAndResolvePrompt(step, context);
991
587
  resolvedInputs = resolveTemplates(resolvedInputs, context);
992
588
  }
993
589
  else {
994
- // Resolve templates in inputs
995
590
  resolvedInputs = resolveTemplates(step.inputs, context);
996
591
  }
997
592
  const stepWithResolvedInputs = { ...step, inputs: resolvedInputs };
998
- // Check if this is a built-in operation
999
593
  let output;
1000
- if (isBuiltInOperation(step.action)) {
1001
- // Execute built-in operation directly (no timeout, no SDK executor needed)
1002
- // For built-in operations, pass both resolved and unresolved inputs
1003
- // to allow selective resolution of template expressions
1004
- // Await in case operation is async (e.g., file operations)
594
+ if (isParallelOperation(step.action)) {
595
+ // Pass both resolved and raw inputs to parallel operations.
596
+ // Resolved inputs are used for structural fields (items, agents, etc.).
597
+ // Raw inputs preserve prompt templates with per-item variables
598
+ // ({{ item }}, {{ itemIndex }}) that can't be resolved until iteration.
599
+ output = await executeParallelOperation(step.action, resolvedInputs, context, sdkRegistry, stepExecutor, step.inputs);
600
+ }
601
+ else if (isBuiltInOperation(step.action)) {
1005
602
  output = await executeBuiltInOperation(step.action, step.inputs, resolvedInputs, context);
1006
603
  }
1007
604
  else {
1008
- // Execute step with executor context
1009
605
  output = await this.executeWithTimeout(() => stepExecutor(stepWithResolvedInputs, context, sdkRegistry, executorContext), step.timeout ?? this.config.defaultTimeout);
1010
606
  }
1011
607
  circuitBreaker.recordSuccess();
@@ -1017,16 +613,13 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
1017
613
  lastError = error instanceof Error ? error : new Error(String(error));
1018
614
  circuitBreaker.recordFailure();
1019
615
  this.events.onStepError?.(step, lastError, attempt);
1020
- // Wait before retry (unless last attempt)
1021
616
  if (attempt < maxRetries) {
1022
617
  const delay = this.retryPolicy.getDelay(attempt);
1023
618
  await sleep(delay);
1024
619
  }
1025
620
  }
1026
621
  }
1027
- // All retries exhausted
1028
- const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, maxRetries, lastError // Pass full error object to preserve HTTP details, stack traces, etc.
1029
- );
622
+ const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, maxRetries, lastError);
1030
623
  this.events.onStepComplete?.(step, result);
1031
624
  return result;
1032
625
  }
@@ -1035,7 +628,6 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
1035
628
  */
1036
629
  async executeStepWithFailover(step, context, sdkRegistry, stepExecutor) {
1037
630
  const primaryResult = await this.executeStepWithRetry(step, context, sdkRegistry, stepExecutor);
1038
- // Sub-workflows and non-action steps don't support failover
1039
631
  if (!isActionStep(step)) {
1040
632
  return primaryResult;
1041
633
  }
@@ -1084,121 +676,42 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
1084
676
  this.healthTracker.markUnhealthy(primaryTool, errorMessage);
1085
677
  return primaryResult;
1086
678
  }
679
+ // ============================================================================
680
+ // Utility Methods
681
+ // ============================================================================
1087
682
  /**
1088
683
  * Execute a function with a timeout.
684
+ * Uses a settled guard to prevent timer leaks and double-resolution.
1089
685
  */
1090
686
  async executeWithTimeout(fn, timeoutMs) {
1091
- return Promise.race([
1092
- fn(),
1093
- new Promise((_, reject) => setTimeout(() => reject(new Error(`Step timed out after ${timeoutMs}ms`)), timeoutMs)),
1094
- ]);
1095
- }
1096
- /**
1097
- * Evaluate step conditions.
1098
- */
1099
- evaluateConditions(conditions, context) {
1100
- for (const condition of conditions) {
1101
- if (!this.evaluateCondition(condition, context)) {
1102
- return false;
1103
- }
1104
- }
1105
- return true;
1106
- }
1107
- /**
1108
- * Evaluate a single condition.
1109
- * Supports: ==, !=, >, <, >=, <=
1110
- * Also supports nested property access (e.g., check_result.success)
1111
- * and step status checks (e.g., step_id.status == 'failed')
1112
- */
1113
- evaluateCondition(condition, context) {
1114
- // Simple expression parsing
1115
- const operators = ['==', '!=', '>=', '<=', '>', '<'];
1116
- let operator;
1117
- let parts = [];
1118
- for (const op of operators) {
1119
- if (condition.includes(op)) {
1120
- operator = op;
1121
- parts = condition.split(op).map((s) => s.trim());
1122
- break;
1123
- }
1124
- }
1125
- if (!operator || parts.length !== 2) {
1126
- // Treat as boolean variable reference with nested property support
1127
- const value = this.resolveConditionValue(condition, context);
1128
- return Boolean(value);
1129
- }
1130
- const left = this.resolveConditionValue(parts[0], context);
1131
- const right = this.parseValue(parts[1]);
1132
- switch (operator) {
1133
- case '==':
1134
- return left == right;
1135
- case '!=':
1136
- return left != right;
1137
- case '>':
1138
- return Number(left) > Number(right);
1139
- case '<':
1140
- return Number(left) < Number(right);
1141
- case '>=':
1142
- return Number(left) >= Number(right);
1143
- case '<=':
1144
- return Number(left) <= Number(right);
1145
- default:
1146
- return false;
1147
- }
1148
- }
1149
- /**
1150
- * Resolve a condition value with support for nested properties.
1151
- * Handles direct variable references and nested paths.
1152
- * Uses Nunjucks for template expressions with filters/regex.
1153
- */
1154
- resolveConditionValue(path, context) {
1155
- // If it looks like a template expression, resolve it
1156
- if (path.includes('|') || path.includes('=~') || path.includes('!~')) {
1157
- // Build template context
1158
- const templateContext = {
1159
- inputs: context.inputs,
1160
- ...context.variables,
1161
- };
1162
- return renderTemplate(`{{ ${path} }}`, templateContext);
1163
- }
1164
- // First try to parse as a literal value (true, false, numbers, etc.)
1165
- const parsedValue = this.parseValue(path);
1166
- // If parseValue returned the same string, try to resolve as a variable
1167
- if (parsedValue === path) {
1168
- const resolved = resolveVariablePath(path, context);
1169
- return resolved;
1170
- }
1171
- // Return the parsed literal value
1172
- return parsedValue;
1173
- }
1174
- /**
1175
- * Parse a value from a condition string.
1176
- */
1177
- parseValue(value) {
1178
- // Remove quotes
1179
- if ((value.startsWith('"') && value.endsWith('"')) ||
1180
- (value.startsWith("'") && value.endsWith("'"))) {
1181
- return value.slice(1, -1);
1182
- }
1183
- // Numbers
1184
- if (!isNaN(Number(value))) {
1185
- return Number(value);
1186
- }
1187
- // Booleans
1188
- if (value === 'true')
1189
- return true;
1190
- if (value === 'false')
1191
- return false;
1192
- if (value === 'null')
1193
- return null;
1194
- return value;
687
+ return new Promise((resolve, reject) => {
688
+ let settled = false;
689
+ const timer = setTimeout(() => {
690
+ if (!settled) {
691
+ settled = true;
692
+ reject(new Error(`Step timed out after ${timeoutMs}ms`));
693
+ }
694
+ }, timeoutMs);
695
+ fn().then((result) => {
696
+ if (!settled) {
697
+ settled = true;
698
+ clearTimeout(timer);
699
+ resolve(result);
700
+ }
701
+ }, (error) => {
702
+ if (!settled) {
703
+ settled = true;
704
+ clearTimeout(timer);
705
+ reject(error);
706
+ }
707
+ });
708
+ });
1195
709
  }
1196
710
  /**
1197
711
  * Build the final workflow result.
1198
712
  */
1199
713
  buildWorkflowResult(workflow, context, stepResults, startedAt, error) {
1200
714
  const completedAt = new Date();
1201
- // Use workflowOutputs if set by workflow.set_outputs, otherwise use all variables
1202
715
  const output = context.workflowOutputs || context.variables;
1203
716
  return {
1204
717
  workflowId: workflow.metadata.id,
@@ -1220,665 +733,6 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
1220
733
  breaker.reset();
1221
734
  }
1222
735
  }
1223
- // ============================================================================
1224
- // Control Flow Execution Methods
1225
- // ============================================================================
1226
- /**
1227
- * Execute an if/else conditional step.
1228
- */
1229
- async executeIfStep(step, context, sdkRegistry, stepExecutor) {
1230
- const startedAt = new Date();
1231
- try {
1232
- // Evaluate condition
1233
- const conditionResult = this.evaluateCondition(step.condition, context);
1234
- // Determine which branch to execute
1235
- const branchSteps = conditionResult
1236
- ? step.then || step.steps // 'steps' is alias for 'then'
1237
- : step.else;
1238
- if (!branchSteps || branchSteps.length === 0) {
1239
- return createStepResult(step.id, StepStatus.SKIPPED, null, startedAt);
1240
- }
1241
- // Execute the branch steps
1242
- const branchResults = [];
1243
- for (const branchStep of branchSteps) {
1244
- const result = await this.executeStep(branchStep, context, sdkRegistry, stepExecutor);
1245
- if (result.status === StepStatus.COMPLETED && branchStep.outputVariable) {
1246
- context.variables[branchStep.outputVariable] = result.output;
1247
- branchResults.push(result.output);
1248
- }
1249
- if (result.status === StepStatus.FAILED) {
1250
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
1251
- }
1252
- }
1253
- return createStepResult(step.id, StepStatus.COMPLETED, branchResults, startedAt);
1254
- }
1255
- catch (error) {
1256
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1257
- }
1258
- }
1259
- /**
1260
- * Execute a switch/case step.
1261
- */
1262
- async executeSwitchStep(step, context, sdkRegistry, stepExecutor) {
1263
- const startedAt = new Date();
1264
- try {
1265
- // Resolve the switch expression
1266
- const expressionValue = String(resolveTemplates(step.expression, context));
1267
- // Find matching case
1268
- const caseSteps = step.cases[expressionValue] || step.default;
1269
- if (!caseSteps || caseSteps.length === 0) {
1270
- return createStepResult(step.id, StepStatus.SKIPPED, null, startedAt);
1271
- }
1272
- // Execute case steps
1273
- const caseResults = [];
1274
- for (const caseStep of caseSteps) {
1275
- const result = await this.executeStep(caseStep, context, sdkRegistry, stepExecutor);
1276
- if (result.status === StepStatus.COMPLETED && caseStep.outputVariable) {
1277
- context.variables[caseStep.outputVariable] = result.output;
1278
- caseResults.push(result.output);
1279
- }
1280
- if (result.status === StepStatus.FAILED) {
1281
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
1282
- }
1283
- }
1284
- return createStepResult(step.id, StepStatus.COMPLETED, caseResults, startedAt);
1285
- }
1286
- catch (error) {
1287
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1288
- }
1289
- }
1290
- /**
1291
- * Execute a for-each loop step.
1292
- * Supports optional batch_size and pause_between_batches for rate limiting.
1293
- */
1294
- async executeForEachStep(step, context, sdkRegistry, stepExecutor) {
1295
- const startedAt = new Date();
1296
- const cleanupLoopVars = () => {
1297
- delete context.variables[step.itemVariable];
1298
- delete context.variables['loop'];
1299
- delete context.variables['batch'];
1300
- if (step.indexVariable)
1301
- delete context.variables[step.indexVariable];
1302
- };
1303
- try {
1304
- // Resolve items array
1305
- const items = resolveTemplates(step.items, context);
1306
- if (!Array.isArray(items)) {
1307
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Items must be an array');
1308
- }
1309
- if (items.length === 0) {
1310
- return createStepResult(step.id, StepStatus.SKIPPED, [], startedAt);
1311
- }
1312
- const batchSize = step.batchSize;
1313
- const pauseBetweenBatches = step.pauseBetweenBatches;
1314
- // If batch mode, process items in batches
1315
- if (batchSize && batchSize > 0) {
1316
- return await this.executeForEachBatched(step, items, batchSize, pauseBetweenBatches ?? 0, context, sdkRegistry, stepExecutor, startedAt);
1317
- }
1318
- // Standard item-by-item execution
1319
- const results = [];
1320
- for (let i = 0; i < items.length; i++) {
1321
- context.variables[step.itemVariable] = items[i];
1322
- context.variables['loop'] = {
1323
- index: i,
1324
- first: i === 0,
1325
- last: i === items.length - 1,
1326
- length: items.length,
1327
- };
1328
- if (step.indexVariable) {
1329
- context.variables[step.indexVariable] = i;
1330
- }
1331
- // Execute iteration steps
1332
- for (const iterStep of step.steps) {
1333
- const result = await this.executeStep(iterStep, context, sdkRegistry, stepExecutor);
1334
- if (result.status === StepStatus.COMPLETED && iterStep.outputVariable) {
1335
- context.variables[iterStep.outputVariable] = result.output;
1336
- }
1337
- if (result.status === StepStatus.FAILED) {
1338
- const errorAction = step.errorHandling?.action ?? 'stop';
1339
- if (errorAction === 'stop') {
1340
- cleanupLoopVars();
1341
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
1342
- }
1343
- break;
1344
- }
1345
- }
1346
- results.push(context.variables[step.itemVariable]);
1347
- }
1348
- cleanupLoopVars();
1349
- return createStepResult(step.id, StepStatus.COMPLETED, results, startedAt);
1350
- }
1351
- catch (error) {
1352
- cleanupLoopVars();
1353
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1354
- }
1355
- }
1356
- /**
1357
- * Execute for-each in batch mode.
1358
- * Items are split into batches; {{ batch }} contains the current batch array.
1359
- */
1360
- async executeForEachBatched(step, items, batchSize, pauseBetweenBatches, context, sdkRegistry, stepExecutor, startedAt) {
1361
- const results = [];
1362
- const totalBatches = Math.ceil(items.length / batchSize);
1363
- for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
1364
- const batchStart = batchIndex * batchSize;
1365
- const batchItems = items.slice(batchStart, batchStart + batchSize);
1366
- // Pause between batches (not before first batch)
1367
- if (batchIndex > 0 && pauseBetweenBatches > 0) {
1368
- await new Promise((resolve) => setTimeout(resolve, pauseBetweenBatches));
1369
- }
1370
- // Expose batch-level variables
1371
- context.variables['batch'] = batchItems;
1372
- context.variables['loop'] = {
1373
- index: batchIndex,
1374
- first: batchIndex === 0,
1375
- last: batchIndex === totalBatches - 1,
1376
- length: totalBatches,
1377
- batchSize,
1378
- batchStart,
1379
- totalItems: items.length,
1380
- };
1381
- // Process each item in the batch
1382
- for (let i = 0; i < batchItems.length; i++) {
1383
- const globalIndex = batchStart + i;
1384
- context.variables[step.itemVariable] = batchItems[i];
1385
- if (step.indexVariable) {
1386
- context.variables[step.indexVariable] = globalIndex;
1387
- }
1388
- for (const iterStep of step.steps) {
1389
- const result = await this.executeStep(iterStep, context, sdkRegistry, stepExecutor);
1390
- if (result.status === StepStatus.COMPLETED && iterStep.outputVariable) {
1391
- context.variables[iterStep.outputVariable] = result.output;
1392
- }
1393
- if (result.status === StepStatus.FAILED) {
1394
- const errorAction = step.errorHandling?.action ?? 'stop';
1395
- if (errorAction === 'stop') {
1396
- delete context.variables[step.itemVariable];
1397
- delete context.variables['loop'];
1398
- delete context.variables['batch'];
1399
- if (step.indexVariable)
1400
- delete context.variables[step.indexVariable];
1401
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
1402
- }
1403
- break;
1404
- }
1405
- }
1406
- results.push(context.variables[step.itemVariable]);
1407
- }
1408
- }
1409
- delete context.variables[step.itemVariable];
1410
- delete context.variables['loop'];
1411
- delete context.variables['batch'];
1412
- if (step.indexVariable)
1413
- delete context.variables[step.indexVariable];
1414
- return createStepResult(step.id, StepStatus.COMPLETED, results, startedAt);
1415
- }
1416
- /**
1417
- * Execute a while loop step.
1418
- */
1419
- async executeWhileStep(step, context, sdkRegistry, stepExecutor) {
1420
- const startedAt = new Date();
1421
- let iterations = 0;
1422
- try {
1423
- while (this.evaluateCondition(step.condition, context)) {
1424
- if (iterations >= step.maxIterations) {
1425
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Max iterations (${step.maxIterations}) exceeded`);
1426
- }
1427
- // Execute iteration steps
1428
- for (const iterStep of step.steps) {
1429
- const result = await this.executeStep(iterStep, context, sdkRegistry, stepExecutor);
1430
- if (result.status === StepStatus.COMPLETED && iterStep.outputVariable) {
1431
- context.variables[iterStep.outputVariable] = result.output;
1432
- }
1433
- if (result.status === StepStatus.FAILED) {
1434
- const errorAction = step.errorHandling?.action ?? 'stop';
1435
- if (errorAction === 'stop') {
1436
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
1437
- }
1438
- // 'continue' - skip to next iteration
1439
- break;
1440
- }
1441
- }
1442
- iterations++;
1443
- }
1444
- return createStepResult(step.id, StepStatus.COMPLETED, { iterations }, startedAt);
1445
- }
1446
- catch (error) {
1447
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1448
- }
1449
- }
1450
- /**
1451
- * Execute a map transformation step.
1452
- */
1453
- async executeMapStep(step, context, _sdkRegistry, _stepExecutor) {
1454
- const startedAt = new Date();
1455
- try {
1456
- // Resolve items array
1457
- const items = resolveTemplates(step.items, context);
1458
- if (!Array.isArray(items)) {
1459
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Items must be an array');
1460
- }
1461
- // Map each item using the expression
1462
- const mapped = items.map((item) => {
1463
- context.variables[step.itemVariable] = item;
1464
- const result = resolveTemplates(step.expression, context);
1465
- delete context.variables[step.itemVariable];
1466
- return result;
1467
- });
1468
- return createStepResult(step.id, StepStatus.COMPLETED, mapped, startedAt);
1469
- }
1470
- catch (error) {
1471
- delete context.variables[step.itemVariable];
1472
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1473
- }
1474
- }
1475
- /**
1476
- * Execute a filter step.
1477
- */
1478
- async executeFilterStep(step, context, _sdkRegistry, _stepExecutor) {
1479
- const startedAt = new Date();
1480
- try {
1481
- // Resolve items array
1482
- const items = resolveTemplates(step.items, context);
1483
- if (!Array.isArray(items)) {
1484
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Items must be an array');
1485
- }
1486
- // Filter items using the condition
1487
- const filtered = items.filter((item) => {
1488
- context.variables[step.itemVariable] = item;
1489
- const result = this.evaluateCondition(step.condition, context);
1490
- delete context.variables[step.itemVariable];
1491
- return result;
1492
- });
1493
- return createStepResult(step.id, StepStatus.COMPLETED, filtered, startedAt);
1494
- }
1495
- catch (error) {
1496
- delete context.variables[step.itemVariable];
1497
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1498
- }
1499
- }
1500
- /**
1501
- * Execute a reduce/aggregate step.
1502
- */
1503
- async executeReduceStep(step, context, _sdkRegistry, _stepExecutor) {
1504
- const startedAt = new Date();
1505
- try {
1506
- // Resolve items array
1507
- const items = resolveTemplates(step.items, context);
1508
- if (!Array.isArray(items)) {
1509
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Items must be an array');
1510
- }
1511
- // Reduce items using the expression
1512
- let accumulator = step.initialValue ?? null;
1513
- for (const item of items) {
1514
- context.variables[step.itemVariable] = item;
1515
- context.variables[step.accumulatorVariable] = accumulator;
1516
- accumulator = resolveTemplates(step.expression, context);
1517
- delete context.variables[step.itemVariable];
1518
- delete context.variables[step.accumulatorVariable];
1519
- }
1520
- return createStepResult(step.id, StepStatus.COMPLETED, accumulator, startedAt);
1521
- }
1522
- catch (error) {
1523
- delete context.variables[step.itemVariable];
1524
- delete context.variables[step.accumulatorVariable];
1525
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1526
- }
1527
- }
1528
- /**
1529
- * Execute parallel branches.
1530
- */
1531
- async executeParallelStep(step, context, sdkRegistry, stepExecutor) {
1532
- const startedAt = new Date();
1533
- try {
1534
- // Execute branches in parallel
1535
- const branchPromises = step.branches.map(async (branch) => {
1536
- // Clone context for isolation
1537
- const branchContext = this.cloneContext(context);
1538
- // Execute branch steps
1539
- const branchResults = [];
1540
- for (const branchStep of branch.steps) {
1541
- const result = await this.executeStep(branchStep, branchContext, sdkRegistry, stepExecutor);
1542
- if (result.status === StepStatus.COMPLETED && branchStep.outputVariable) {
1543
- branchContext.variables[branchStep.outputVariable] = result.output;
1544
- branchResults.push(result.output);
1545
- }
1546
- if (result.status === StepStatus.FAILED) {
1547
- throw new Error(`Branch ${branch.id} failed: ${errorToString(result.error)}`);
1548
- }
1549
- }
1550
- return { branchId: branch.id, context: branchContext, results: branchResults };
1551
- });
1552
- // Wait for all branches (or limited concurrency)
1553
- const branchResults = step.maxConcurrent
1554
- ? await this.executeConcurrentlyWithLimit(branchPromises, step.maxConcurrent)
1555
- : await Promise.all(branchPromises);
1556
- // Merge branch contexts back into main context
1557
- for (const { branchId, context: branchContext } of branchResults) {
1558
- this.mergeContexts(context, branchContext, branchId);
1559
- }
1560
- const outputs = branchResults.map((br) => br.results);
1561
- return createStepResult(step.id, StepStatus.COMPLETED, outputs, startedAt);
1562
- }
1563
- catch (error) {
1564
- if (step.onError === 'continue') {
1565
- return createStepResult(step.id, StepStatus.COMPLETED, null, startedAt);
1566
- }
1567
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1568
- }
1569
- }
1570
- /**
1571
- * Execute try/catch/finally step.
1572
- */
1573
- async executeTryStep(step, context, sdkRegistry, stepExecutor) {
1574
- const startedAt = new Date();
1575
- let tryError;
1576
- try {
1577
- // Execute try block
1578
- for (const tryStep of step.try) {
1579
- const result = await this.executeStep(tryStep, context, sdkRegistry, stepExecutor);
1580
- if (result.status === StepStatus.COMPLETED && tryStep.outputVariable) {
1581
- context.variables[tryStep.outputVariable] = result.output;
1582
- }
1583
- if (result.status === StepStatus.FAILED) {
1584
- tryError = new Error(result.error ? errorToString(result.error) : 'Step failed');
1585
- break;
1586
- }
1587
- }
1588
- // If error occurred and catch block exists, execute catch
1589
- let catchError;
1590
- if (tryError && step.catch) {
1591
- // Inject error object into context
1592
- context.variables['error'] = {
1593
- message: tryError.message,
1594
- step: tryError,
1595
- };
1596
- for (const catchStep of step.catch) {
1597
- const result = await this.executeStep(catchStep, context, sdkRegistry, stepExecutor);
1598
- if (result.status === StepStatus.COMPLETED && catchStep.outputVariable) {
1599
- context.variables[catchStep.outputVariable] = result.output;
1600
- }
1601
- if (result.status === StepStatus.FAILED) {
1602
- catchError = new Error(result.error ? errorToString(result.error) : 'Catch block failed');
1603
- break;
1604
- }
1605
- }
1606
- delete context.variables['error'];
1607
- }
1608
- // Execute finally block (always runs)
1609
- if (step.finally) {
1610
- for (const finallyStep of step.finally) {
1611
- const result = await this.executeStep(finallyStep, context, sdkRegistry, stepExecutor);
1612
- if (result.status === StepStatus.COMPLETED && finallyStep.outputVariable) {
1613
- context.variables[finallyStep.outputVariable] = result.output;
1614
- }
1615
- }
1616
- }
1617
- // Return success if catch handled the error, or error if not
1618
- if (tryError && !step.catch) {
1619
- // No catch block to handle error
1620
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, tryError.message);
1621
- }
1622
- if (catchError) {
1623
- // Catch block also failed
1624
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, catchError.message);
1625
- }
1626
- return createStepResult(step.id, StepStatus.COMPLETED, null, startedAt);
1627
- }
1628
- catch (error) {
1629
- // Execute finally even on unexpected error
1630
- if (step.finally) {
1631
- try {
1632
- for (const finallyStep of step.finally) {
1633
- const result = await this.executeStep(finallyStep, context, sdkRegistry, stepExecutor);
1634
- if (result.status === StepStatus.COMPLETED && finallyStep.outputVariable) {
1635
- context.variables[finallyStep.outputVariable] = result.output;
1636
- }
1637
- }
1638
- }
1639
- catch {
1640
- // Ignore finally errors
1641
- }
1642
- }
1643
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1644
- }
1645
- }
1646
- /**
1647
- * Execute a script step (inline JavaScript).
1648
- */
1649
- async executeScriptStep(step, context) {
1650
- const startedAt = new Date();
1651
- try {
1652
- // Resolve any templates in the code
1653
- const resolvedInputs = resolveTemplates(step.inputs, context);
1654
- // Execute the script with the workflow context
1655
- const result = await executeScriptAsync(resolvedInputs.code, {
1656
- variables: context.variables,
1657
- inputs: context.inputs,
1658
- steps: context.stepMetadata,
1659
- }, {
1660
- timeout: resolvedInputs.timeout ?? 5000,
1661
- });
1662
- if (!result.success) {
1663
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error ?? 'Script execution failed');
1664
- }
1665
- return createStepResult(step.id, StepStatus.COMPLETED, result.value, startedAt);
1666
- }
1667
- catch (error) {
1668
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1669
- }
1670
- }
1671
- // ============================================================================
1672
- // Wait Step Execution
1673
- // ============================================================================
1674
- /**
1675
- * Execute a wait/pause step.
1676
- *
1677
- * For mode=duration: In-process wait (suitable for short durations).
1678
- * For persistent long-duration waits, a WaitManager should checkpoint
1679
- * the execution and resume it later via the scheduler.
1680
- *
1681
- * For mode=webhook: Returns a resume URL. The execution will be
1682
- * checkpointed and resumed when the URL is called.
1683
- *
1684
- * For mode=form: Returns form fields. The execution will be
1685
- * checkpointed and resumed when the form is submitted.
1686
- */
1687
- async executeWaitStep(step, context) {
1688
- const startedAt = new Date();
1689
- try {
1690
- switch (step.mode) {
1691
- case 'duration': {
1692
- if (!step.duration) {
1693
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Wait step with mode=duration requires a duration');
1694
- }
1695
- const resolvedDuration = resolveTemplates(step.duration, context);
1696
- const ms = parseDuration(resolvedDuration);
1697
- // For short durations (under 5 minutes), do in-process wait
1698
- if (ms <= 300000) {
1699
- await new Promise((resolve) => setTimeout(resolve, ms));
1700
- return createStepResult(step.id, StepStatus.COMPLETED, { waited: ms }, startedAt);
1701
- }
1702
- // For longer durations, checkpoint and schedule resume
1703
- // The StateStore will persist the execution state
1704
- if (this.stateStore) {
1705
- this.stateStore.saveCheckpoint({
1706
- runId: context.runId,
1707
- stepIndex: context.currentStepIndex,
1708
- stepName: step.id,
1709
- status: StepStatus.COMPLETED,
1710
- startedAt: startedAt,
1711
- completedAt: new Date(),
1712
- inputs: { mode: 'duration', resumeAt: new Date(Date.now() + ms).toISOString() },
1713
- outputs: { waiting: true },
1714
- error: null,
1715
- retryCount: 0,
1716
- });
1717
- }
1718
- // Set execution status to indicate waiting
1719
- const resumeAt = new Date(Date.now() + ms).toISOString();
1720
- return createStepResult(step.id, StepStatus.COMPLETED, {
1721
- waiting: true,
1722
- mode: 'duration',
1723
- resumeAt,
1724
- durationMs: ms,
1725
- }, startedAt);
1726
- }
1727
- case 'webhook': {
1728
- // Generate a unique resume URL
1729
- const resumeToken = crypto.randomUUID();
1730
- const webhookPath = step.webhookPath
1731
- ? resolveTemplates(step.webhookPath, context)
1732
- : `/resume/${context.runId}/${step.id}/${resumeToken}`;
1733
- if (this.stateStore) {
1734
- this.stateStore.saveCheckpoint({
1735
- runId: context.runId,
1736
- stepIndex: context.currentStepIndex,
1737
- stepName: step.id,
1738
- status: StepStatus.COMPLETED,
1739
- startedAt: startedAt,
1740
- completedAt: new Date(),
1741
- inputs: { mode: 'webhook', resumeToken, webhookPath },
1742
- outputs: { waiting: true },
1743
- error: null,
1744
- retryCount: 0,
1745
- });
1746
- }
1747
- return createStepResult(step.id, StepStatus.COMPLETED, {
1748
- waiting: true,
1749
- mode: 'webhook',
1750
- resumeToken,
1751
- webhookPath,
1752
- }, startedAt);
1753
- }
1754
- case 'form': {
1755
- if (!step.fields || Object.keys(step.fields).length === 0) {
1756
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Wait step with mode=form requires fields');
1757
- }
1758
- const resumeToken = crypto.randomUUID();
1759
- if (this.stateStore) {
1760
- this.stateStore.saveCheckpoint({
1761
- runId: context.runId,
1762
- stepIndex: context.currentStepIndex,
1763
- stepName: step.id,
1764
- status: StepStatus.COMPLETED,
1765
- startedAt: startedAt,
1766
- completedAt: new Date(),
1767
- inputs: { mode: 'form', resumeToken, fields: step.fields },
1768
- outputs: { waiting: true },
1769
- error: null,
1770
- retryCount: 0,
1771
- });
1772
- }
1773
- return createStepResult(step.id, StepStatus.COMPLETED, {
1774
- waiting: true,
1775
- mode: 'form',
1776
- resumeToken,
1777
- fields: step.fields,
1778
- formPath: `/form/${context.runId}/${step.id}/${resumeToken}`,
1779
- }, startedAt);
1780
- }
1781
- default:
1782
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Unknown wait mode: ${step.mode}`);
1783
- }
1784
- }
1785
- catch (error) {
1786
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1787
- }
1788
- }
1789
- // ============================================================================
1790
- // Merge Step Execution
1791
- // ============================================================================
1792
- /**
1793
- * Execute a merge step - combines data from multiple sources.
1794
- */
1795
- async executeMergeStep(step, context) {
1796
- const startedAt = new Date();
1797
- try {
1798
- // Resolve all source expressions to arrays
1799
- const resolvedSources = [];
1800
- for (const source of step.sources) {
1801
- const resolved = resolveTemplates(source, context);
1802
- if (!Array.isArray(resolved)) {
1803
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Merge source "${source}" did not resolve to an array`);
1804
- }
1805
- resolvedSources.push(resolved);
1806
- }
1807
- let result;
1808
- switch (step.mode) {
1809
- case 'append':
1810
- result = resolvedSources.flat();
1811
- break;
1812
- case 'match': {
1813
- if (!step.matchField) {
1814
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "match" requires matchField');
1815
- }
1816
- // Return items that appear in ALL sources (by matchField)
1817
- const fieldSets = resolvedSources.map((source) => new Set(source.map((item) => getField(item, step.matchField))));
1818
- // Intersection of all field sets
1819
- const commonKeys = fieldSets.reduce((acc, set) => new Set([...acc].filter((key) => set.has(key))));
1820
- // Return items from first source that match
1821
- result = resolvedSources[0].filter((item) => commonKeys.has(getField(item, step.matchField)));
1822
- break;
1823
- }
1824
- case 'diff': {
1825
- if (!step.matchField) {
1826
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "diff" requires matchField');
1827
- }
1828
- // Return items from first source NOT found in other sources
1829
- const otherKeys = new Set(resolvedSources.slice(1).flat().map((item) => getField(item, step.matchField)));
1830
- result = resolvedSources[0].filter((item) => !otherKeys.has(getField(item, step.matchField)));
1831
- break;
1832
- }
1833
- case 'combine_by_field': {
1834
- if (!step.matchField) {
1835
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "combine_by_field" requires matchField');
1836
- }
1837
- // Group items by matchField and merge their properties
1838
- const grouped = new Map();
1839
- const onConflict = step.onConflict ?? 'keep_last';
1840
- for (const source of resolvedSources) {
1841
- for (const item of source) {
1842
- if (!item || typeof item !== 'object')
1843
- continue;
1844
- const key = getField(item, step.matchField);
1845
- const existing = grouped.get(key);
1846
- if (existing) {
1847
- if (onConflict === 'keep_first') {
1848
- // Only add new fields
1849
- for (const [k, v] of Object.entries(item)) {
1850
- if (!(k in existing))
1851
- existing[k] = v;
1852
- }
1853
- }
1854
- else if (onConflict === 'keep_last') {
1855
- Object.assign(existing, item);
1856
- }
1857
- else {
1858
- // merge_fields: deep merge
1859
- Object.assign(existing, item);
1860
- }
1861
- }
1862
- else {
1863
- grouped.set(key, { ...item });
1864
- }
1865
- }
1866
- }
1867
- result = Array.from(grouped.values());
1868
- break;
1869
- }
1870
- default:
1871
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Unknown merge mode: ${step.mode}`);
1872
- }
1873
- return createStepResult(step.id, StepStatus.COMPLETED, result, startedAt);
1874
- }
1875
- catch (error) {
1876
- return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
1877
- }
1878
- }
1879
- // ============================================================================
1880
- // Helper Methods for Control Flow
1881
- // ============================================================================
1882
736
  /**
1883
737
  * Clone execution context for parallel branches.
1884
738
  */
@@ -1894,25 +748,26 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
1894
748
  * Merge branch context back into main context.
1895
749
  */
1896
750
  mergeContexts(mainContext, branchContext, branchId) {
1897
- // Merge variables with branch prefix
1898
751
  for (const [key, value] of Object.entries(branchContext.variables)) {
1899
752
  mainContext.variables[`${branchId}.${key}`] = value;
1900
753
  }
1901
754
  }
1902
755
  /**
1903
756
  * Execute promises with concurrency limit.
757
+ * Results are returned in the same order as the input promises.
1904
758
  */
1905
759
  async executeConcurrentlyWithLimit(promises, limit) {
1906
- const results = [];
1907
- const executing = [];
1908
- for (const promise of promises) {
1909
- const p = promise.then((result) => {
1910
- results.push(result);
760
+ const results = new Array(promises.length);
761
+ const executing = new Set();
762
+ for (let i = 0; i < promises.length; i++) {
763
+ const p = promises[i].then((result) => {
764
+ results[i] = result;
765
+ }).then(() => {
766
+ executing.delete(p);
1911
767
  });
1912
- executing.push(p);
1913
- if (executing.length >= limit) {
768
+ executing.add(p);
769
+ if (executing.size >= limit) {
1914
770
  await Promise.race(executing);
1915
- executing.splice(executing.findIndex((x) => x === p), 1);
1916
771
  }
1917
772
  }
1918
773
  await Promise.all(executing);