@multiplier-labs/stepflow 0.3.1 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codekin/reports/code-review/2026-04-11_code-review-daily.md +146 -0
- package/.codekin/reports/code-review/2026-04-13_code-review-daily.md +74 -0
- package/.codekin/reports/comments/2026-04-04_comment-assessment.md +179 -0
- package/.codekin/reports/comments/2026-04-11_comment-assessment.md +161 -0
- package/.codekin/reports/comments/2026-04-14_comment-assessment.md +34 -0
- package/.codekin/reports/complexity/2026-04-03_complexity-report.md +152 -0
- package/.codekin/reports/complexity/2026-04-11_complexity-report.md +152 -0
- package/.codekin/reports/complexity/2026-04-14_complexity-report.md +34 -0
- package/.codekin/reports/dependencies/2026-04-04_dependency-health.md +94 -0
- package/.codekin/reports/dependencies/2026-04-11_dependency-health.md +93 -0
- package/.codekin/reports/docs-audit/2026-04-04_docs-audit.md +10 -0
- package/.codekin/reports/docs-audit/2026-04-11_docs-audit.md +138 -0
- package/.codekin/reports/docs-audit/2026-04-13_docs-audit.md +121 -0
- package/.codekin/reports/security/2026-04-03_security-audit.md +208 -0
- package/.codekin/reports/security/2026-04-11_security-audit.md +297 -0
- package/.github/CODEOWNERS +15 -0
- package/.github/dependabot.yml +23 -0
- package/.github/workflows/ci.yml +38 -0
- package/.github/workflows/publish.yml +17 -5
- package/.github/workflows/security.yml +29 -0
- package/CHANGELOG.md +34 -0
- package/README.md +24 -0
- package/SECURITY.md +40 -0
- package/dist/{chunk-FQCT3I2A.js → chunk-5XJBAPXF.js} +527 -551
- package/dist/chunk-5XJBAPXF.js.map +1 -0
- package/dist/{chunk-U5QGN3RT.js → chunk-FMPJBK2N.js} +67 -55
- package/dist/chunk-FMPJBK2N.js.map +1 -0
- package/dist/{chunk-GXY66V7Q.js → chunk-JPXPB7U5.js} +52 -2
- package/dist/chunk-JPXPB7U5.js.map +1 -0
- package/dist/{chunk-XCF4PLN7.js → chunk-U3PCVK6O.js} +276 -61
- package/dist/chunk-U3PCVK6O.js.map +1 -0
- package/dist/events/index.d.ts +60 -7
- package/dist/events/index.js +1 -1
- package/dist/{index-CPfGsmrp.d.ts → index-Cd0IZ40L.d.ts} +56 -17
- package/dist/index.d.ts +15 -23
- package/dist/index.js +784 -547
- package/dist/index.js.map +1 -1
- package/dist/scheduler/index.d.ts +4 -4
- package/dist/scheduler/index.js +2 -2
- package/dist/storage/index.d.ts +322 -56
- package/dist/storage/index.js +4 -2
- package/dist/{types-BlwFEGzj.d.ts → types-DSEjiLMj.d.ts} +16 -15
- package/dist/{types-DzkEqnQJ.d.ts → types-DZUdWmCc.d.ts} +2 -2
- package/dist/{types-BVShq7zT.d.ts → types-K5Gjk3H_.d.ts} +4 -4
- package/docs/stepflow-api-reference.md +5 -4
- package/package.json +12 -8
- package/src/core/engine/concurrency.ts +229 -0
- package/src/core/engine/lifecycle.ts +176 -0
- package/src/core/engine.test.ts +536 -522
- package/src/core/engine.ts +116 -235
- package/src/core/index.ts +3 -3
- package/src/core/orchestrator/persistence.ts +165 -0
- package/src/core/orchestrator/plan-resolution.ts +140 -0
- package/src/core/orchestrator/state-transitions.ts +109 -0
- package/src/core/orchestrator/step-runner.ts +427 -0
- package/src/core/orchestrator.ts +57 -395
- package/src/core/types.ts +7 -29
- package/src/events/index.ts +4 -4
- package/src/events/memory.test.ts +39 -41
- package/src/events/memory.ts +7 -10
- package/src/events/socketio.test.ts +112 -162
- package/src/events/socketio.ts +20 -27
- package/src/events/types.ts +16 -19
- package/src/events/webhook.test.ts +489 -235
- package/src/events/webhook.ts +355 -139
- package/src/index.ts +19 -19
- package/src/planning/evaluate-condition.test.ts +153 -0
- package/src/planning/evaluate-condition.ts +105 -0
- package/src/planning/index.ts +6 -3
- package/src/planning/planner.ts +45 -152
- package/src/planning/planning.test.ts +303 -366
- package/src/planning/registry.test.ts +204 -251
- package/src/planning/registry.ts +20 -106
- package/src/planning/types.ts +23 -27
- package/src/scheduler/cron.test.ts +149 -150
- package/src/scheduler/cron.ts +58 -69
- package/src/scheduler/index.ts +4 -14
- package/src/scheduler/postgres-persistence.ts +89 -105
- package/src/scheduler/sqlite-persistence.test.ts +96 -96
- package/src/scheduler/sqlite-persistence.ts +40 -22
- package/src/scheduler/types.ts +4 -9
- package/src/storage/index.ts +5 -5
- package/src/storage/memory.test.ts +133 -183
- package/src/storage/memory.ts +21 -42
- package/src/storage/postgres-core.ts +550 -0
- package/src/storage/postgres-migration.test.ts +69 -0
- package/src/storage/postgres-transaction-pagination.test.ts +107 -0
- package/src/storage/postgres.ts +222 -937
- package/src/storage/sqlite-core.ts +151 -0
- package/src/storage/sqlite.test.ts +330 -281
- package/src/storage/sqlite.ts +225 -170
- package/src/storage/types.ts +33 -58
- package/src/utils/errors.test.ts +78 -80
- package/src/utils/errors.ts +27 -57
- package/src/utils/id.ts +3 -3
- package/src/utils/index.ts +6 -4
- package/src/utils/logger.test.ts +52 -58
- package/src/utils/logger.ts +4 -4
- package/src/utils/pinned-fetch.test.ts +84 -0
- package/src/utils/pinned-fetch.ts +118 -0
- package/src/utils/postgres-deps.ts +20 -4
- package/src/utils/retry.test.ts +47 -67
- package/src/utils/retry.ts +9 -9
- package/src/utils/safe-json.test.ts +166 -0
- package/src/utils/safe-json.ts +107 -0
- package/src/utils/webhook-signature.test.ts +111 -0
- package/src/utils/webhook-signature.ts +74 -0
- package/dist/chunk-FQCT3I2A.js.map +0 -1
- package/dist/chunk-GXY66V7Q.js.map +0 -1
- package/dist/chunk-U5QGN3RT.js.map +0 -1
- package/dist/chunk-XCF4PLN7.js.map +0 -1
- package/src/scheduler/postgres-persistence.test.ts +0 -906
package/dist/index.js
CHANGED
|
@@ -2,24 +2,24 @@ import {
|
|
|
2
2
|
MemoryStorageAdapter,
|
|
3
3
|
PostgresStorageAdapter,
|
|
4
4
|
SQLiteStorageAdapter
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-5XJBAPXF.js";
|
|
6
6
|
import {
|
|
7
7
|
MemoryEventTransport,
|
|
8
8
|
SocketIOEventTransport,
|
|
9
9
|
WebhookEventTransport
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-U3PCVK6O.js";
|
|
11
11
|
import {
|
|
12
12
|
CronScheduler,
|
|
13
13
|
PostgresSchedulePersistence,
|
|
14
14
|
SQLiteSchedulePersistence
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-FMPJBK2N.js";
|
|
16
16
|
import {
|
|
17
17
|
ConsoleLogger,
|
|
18
18
|
SilentLogger,
|
|
19
19
|
createScopedLogger,
|
|
20
20
|
generateId,
|
|
21
21
|
sanitizeErrorForStorage
|
|
22
|
-
} from "./chunk-
|
|
22
|
+
} from "./chunk-JPXPB7U5.js";
|
|
23
23
|
|
|
24
24
|
// src/utils/errors.ts
|
|
25
25
|
var WorkflowEngineError = class _WorkflowEngineError extends Error {
|
|
@@ -38,7 +38,6 @@ var WorkflowEngineError = class _WorkflowEngineError extends Error {
|
|
|
38
38
|
return {
|
|
39
39
|
code: this.code,
|
|
40
40
|
message: this.message,
|
|
41
|
-
stack: this.stack,
|
|
42
41
|
details: this.details
|
|
43
42
|
};
|
|
44
43
|
}
|
|
@@ -52,8 +51,7 @@ var WorkflowEngineError = class _WorkflowEngineError extends Error {
|
|
|
52
51
|
if (error instanceof Error) {
|
|
53
52
|
return {
|
|
54
53
|
code: defaultCode,
|
|
55
|
-
message: error.message
|
|
56
|
-
stack: error.stack
|
|
54
|
+
message: error.message
|
|
57
55
|
};
|
|
58
56
|
}
|
|
59
57
|
return {
|
|
@@ -64,19 +62,13 @@ var WorkflowEngineError = class _WorkflowEngineError extends Error {
|
|
|
64
62
|
};
|
|
65
63
|
var WorkflowNotFoundError = class extends WorkflowEngineError {
|
|
66
64
|
constructor(kind) {
|
|
67
|
-
super("WORKFLOW_NOT_FOUND", `Workflow "${kind}" is not registered`, {
|
|
68
|
-
kind
|
|
69
|
-
});
|
|
65
|
+
super("WORKFLOW_NOT_FOUND", `Workflow "${kind}" is not registered`, { kind });
|
|
70
66
|
this.name = "WorkflowNotFoundError";
|
|
71
67
|
}
|
|
72
68
|
};
|
|
73
69
|
var WorkflowAlreadyRegisteredError = class extends WorkflowEngineError {
|
|
74
70
|
constructor(kind) {
|
|
75
|
-
super(
|
|
76
|
-
"WORKFLOW_ALREADY_REGISTERED",
|
|
77
|
-
`Workflow "${kind}" is already registered`,
|
|
78
|
-
{ kind }
|
|
79
|
-
);
|
|
71
|
+
super("WORKFLOW_ALREADY_REGISTERED", `Workflow "${kind}" is already registered`, { kind });
|
|
80
72
|
this.name = "WorkflowAlreadyRegisteredError";
|
|
81
73
|
}
|
|
82
74
|
};
|
|
@@ -113,9 +105,7 @@ var StepTimeoutError = class extends WorkflowEngineError {
|
|
|
113
105
|
};
|
|
114
106
|
var WorkflowCanceledError = class extends WorkflowEngineError {
|
|
115
107
|
constructor(runId) {
|
|
116
|
-
super("WORKFLOW_CANCELED", `Workflow run "${runId}" was canceled`, {
|
|
117
|
-
runId
|
|
118
|
-
});
|
|
108
|
+
super("WORKFLOW_CANCELED", `Workflow run "${runId}" was canceled`, { runId });
|
|
119
109
|
this.name = "WorkflowCanceledError";
|
|
120
110
|
}
|
|
121
111
|
};
|
|
@@ -123,14 +113,10 @@ var WaitForRunTimeoutError = class extends WorkflowEngineError {
|
|
|
123
113
|
runId;
|
|
124
114
|
timeoutMs;
|
|
125
115
|
constructor(runId, timeoutMs) {
|
|
126
|
-
super(
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
runId,
|
|
131
|
-
timeoutMs
|
|
132
|
-
}
|
|
133
|
-
);
|
|
116
|
+
super("WAIT_FOR_RUN_TIMEOUT", `Timeout waiting for run ${runId} after ${timeoutMs}ms`, {
|
|
117
|
+
runId,
|
|
118
|
+
timeoutMs
|
|
119
|
+
});
|
|
134
120
|
this.name = "WaitForRunTimeoutError";
|
|
135
121
|
this.runId = runId;
|
|
136
122
|
this.timeoutMs = timeoutMs;
|
|
@@ -139,19 +125,105 @@ var WaitForRunTimeoutError = class extends WorkflowEngineError {
|
|
|
139
125
|
var WorkflowTimeoutError = class extends WorkflowEngineError {
|
|
140
126
|
timeoutMs;
|
|
141
127
|
constructor(runId, timeoutMs) {
|
|
142
|
-
super(
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
runId,
|
|
147
|
-
timeoutMs
|
|
148
|
-
}
|
|
149
|
-
);
|
|
128
|
+
super("WORKFLOW_TIMEOUT", `Workflow run "${runId}" timed out after ${timeoutMs}ms`, {
|
|
129
|
+
runId,
|
|
130
|
+
timeoutMs
|
|
131
|
+
});
|
|
150
132
|
this.name = "WorkflowTimeoutError";
|
|
151
133
|
this.timeoutMs = timeoutMs;
|
|
152
134
|
}
|
|
153
135
|
};
|
|
154
136
|
|
|
137
|
+
// src/core/orchestrator/persistence.ts
|
|
138
|
+
async function markRunRunning(storage, runId, isResume) {
|
|
139
|
+
await storage.updateRun(runId, {
|
|
140
|
+
status: "running",
|
|
141
|
+
...isResume ? {} : { startedAt: /* @__PURE__ */ new Date() }
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
async function saveCheckpoint(storage, runId, results, completedSteps) {
|
|
145
|
+
await storage.updateRun(runId, {
|
|
146
|
+
context: { ...results },
|
|
147
|
+
completedSteps: [...completedSteps]
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
async function markRunSucceeded(storage, runId, results) {
|
|
151
|
+
await storage.updateRun(runId, {
|
|
152
|
+
status: "succeeded",
|
|
153
|
+
context: results,
|
|
154
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
async function markRunFinal(storage, runId, status, results, error) {
|
|
158
|
+
await storage.updateRun(runId, {
|
|
159
|
+
status,
|
|
160
|
+
context: results,
|
|
161
|
+
error,
|
|
162
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
async function createStepRecord(storage, runId, stepKey, stepName, attempt) {
|
|
166
|
+
return storage.createStep({
|
|
167
|
+
runId,
|
|
168
|
+
stepKey,
|
|
169
|
+
stepName,
|
|
170
|
+
status: "running",
|
|
171
|
+
attempt,
|
|
172
|
+
startedAt: /* @__PURE__ */ new Date()
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
async function markStepSucceeded(storage, stepId, result) {
|
|
176
|
+
await storage.updateStep(stepId, {
|
|
177
|
+
status: "succeeded",
|
|
178
|
+
result,
|
|
179
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
async function markStepFailed(storage, stepId, error, isCanceled) {
|
|
183
|
+
await storage.updateStep(stepId, {
|
|
184
|
+
status: isCanceled ? "canceled" : "failed",
|
|
185
|
+
error: WorkflowEngineError.fromError(error),
|
|
186
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
async function markStepCanceled(storage, stepId) {
|
|
190
|
+
await storage.updateStep(stepId, {
|
|
191
|
+
status: "canceled",
|
|
192
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
function safeEmitEvent(events, logger, event) {
|
|
196
|
+
try {
|
|
197
|
+
events.emit(event);
|
|
198
|
+
} catch (error) {
|
|
199
|
+
logger.error("Failed to emit event:", error);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// src/core/orchestrator/state-transitions.ts
|
|
204
|
+
function decideRunFinalState(error) {
|
|
205
|
+
if (error instanceof WorkflowTimeoutError) {
|
|
206
|
+
return { status: "failed", eventType: "run.timeout" };
|
|
207
|
+
}
|
|
208
|
+
if (error instanceof WorkflowCanceledError) {
|
|
209
|
+
return { status: "canceled", eventType: "run.canceled" };
|
|
210
|
+
}
|
|
211
|
+
return { status: "failed", eventType: "run.failed" };
|
|
212
|
+
}
|
|
213
|
+
function resolveStepErrorPolicy(step, definition) {
|
|
214
|
+
return {
|
|
215
|
+
onError: step.onError ?? definition.defaultOnError ?? "fail",
|
|
216
|
+
maxRetries: step.maxRetries ?? 3,
|
|
217
|
+
retryDelay: step.retryDelay ?? 1e3,
|
|
218
|
+
retryBackoff: step.retryBackoff ?? 2
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
function decideStepFailureAction(policy, attempt) {
|
|
222
|
+
if (policy.onError === "skip") return "skip";
|
|
223
|
+
if (policy.onError === "retry" && attempt <= policy.maxRetries) return "retry";
|
|
224
|
+
return "fail";
|
|
225
|
+
}
|
|
226
|
+
|
|
155
227
|
// src/utils/retry.ts
|
|
156
228
|
var DEFAULT_RETRY_OPTIONS = {
|
|
157
229
|
maxRetries: 3,
|
|
@@ -188,7 +260,7 @@ async function withRetry(fn, options = {}) {
|
|
|
188
260
|
let currentDelay = opts.delay;
|
|
189
261
|
for (let attempt = 1; attempt <= opts.maxRetries + 1; attempt++) {
|
|
190
262
|
if (opts.signal?.aborted) {
|
|
191
|
-
throw new
|
|
263
|
+
throw new WorkflowCanceledError("run");
|
|
192
264
|
}
|
|
193
265
|
try {
|
|
194
266
|
return await fn();
|
|
@@ -208,46 +280,241 @@ function calculateRetryDelay(attempt, baseDelay, backoff) {
|
|
|
208
280
|
return Math.round(baseDelay * Math.pow(backoff, attempt - 1));
|
|
209
281
|
}
|
|
210
282
|
|
|
211
|
-
// src/core/orchestrator.ts
|
|
212
|
-
async function
|
|
283
|
+
// src/core/orchestrator/step-runner.ts
|
|
284
|
+
async function executeStep(step, context, options) {
|
|
285
|
+
const { definition, storage, events, logger, abortController } = options;
|
|
286
|
+
const policy = resolveStepErrorPolicy(step, definition);
|
|
287
|
+
let attempt = 0;
|
|
288
|
+
while (true) {
|
|
289
|
+
attempt++;
|
|
290
|
+
throwIfCanceled(abortController, context.runId);
|
|
291
|
+
const stepRecord = await beginStepAttempt(
|
|
292
|
+
storage,
|
|
293
|
+
events,
|
|
294
|
+
logger,
|
|
295
|
+
context,
|
|
296
|
+
step,
|
|
297
|
+
attempt
|
|
298
|
+
);
|
|
299
|
+
if (definition.hooks?.beforeStep) {
|
|
300
|
+
await definition.hooks.beforeStep(context, step);
|
|
301
|
+
}
|
|
302
|
+
try {
|
|
303
|
+
const result = await runStepHandler(step, context, abortController);
|
|
304
|
+
await completeStepAttempt(
|
|
305
|
+
storage,
|
|
306
|
+
events,
|
|
307
|
+
logger,
|
|
308
|
+
stepRecord,
|
|
309
|
+
context,
|
|
310
|
+
step,
|
|
311
|
+
result,
|
|
312
|
+
definition
|
|
313
|
+
);
|
|
314
|
+
return result;
|
|
315
|
+
} catch (error) {
|
|
316
|
+
const lastError = toError(error);
|
|
317
|
+
const isCanceled = error instanceof WorkflowCanceledError || abortController.signal.aborted;
|
|
318
|
+
await markStepFailed(storage, stepRecord.id, error, isCanceled);
|
|
319
|
+
if (isCanceled) {
|
|
320
|
+
emitStepFailedEvent(events, logger, context, step, lastError, attempt);
|
|
321
|
+
throw error;
|
|
322
|
+
}
|
|
323
|
+
await runOnStepErrorHook(definition, context, step, lastError, logger);
|
|
324
|
+
emitStepFailedEvent(events, logger, context, step, lastError, attempt);
|
|
325
|
+
const action = decideStepFailureAction(policy, attempt);
|
|
326
|
+
if (action === "skip") {
|
|
327
|
+
handleSkipAction(events, logger, context, step, lastError);
|
|
328
|
+
return void 0;
|
|
329
|
+
}
|
|
330
|
+
if (action === "retry") {
|
|
331
|
+
await handleRetryAction(
|
|
332
|
+
storage,
|
|
333
|
+
events,
|
|
334
|
+
logger,
|
|
335
|
+
abortController,
|
|
336
|
+
context,
|
|
337
|
+
step,
|
|
338
|
+
attempt,
|
|
339
|
+
policy,
|
|
340
|
+
lastError,
|
|
341
|
+
stepRecord.id
|
|
342
|
+
);
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
throw new StepError(step.key, lastError.message, attempt, lastError);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
function throwIfCanceled(abortController, runId) {
|
|
350
|
+
if (abortController.signal.aborted) {
|
|
351
|
+
throw new WorkflowCanceledError(runId);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
async function beginStepAttempt(storage, events, logger, context, step, attempt) {
|
|
355
|
+
const stepRecord = await createStepRecord(
|
|
356
|
+
storage,
|
|
357
|
+
context.runId,
|
|
358
|
+
step.key,
|
|
359
|
+
step.name,
|
|
360
|
+
attempt
|
|
361
|
+
);
|
|
362
|
+
context.stepId = stepRecord.id;
|
|
363
|
+
safeEmitEvent(events, logger, {
|
|
364
|
+
runId: context.runId,
|
|
365
|
+
kind: context.kind,
|
|
366
|
+
eventType: "step.started",
|
|
367
|
+
stepKey: step.key,
|
|
368
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
369
|
+
payload: { attempt }
|
|
370
|
+
});
|
|
371
|
+
return stepRecord;
|
|
372
|
+
}
|
|
373
|
+
async function runStepHandler(step, context, abortController) {
|
|
374
|
+
if (step.timeout) {
|
|
375
|
+
return executeWithTimeout(
|
|
376
|
+
() => step.handler(context),
|
|
377
|
+
step.timeout,
|
|
378
|
+
abortController.signal,
|
|
379
|
+
step.key
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
return raceWithAbort(step.handler(context), abortController.signal);
|
|
383
|
+
}
|
|
384
|
+
async function completeStepAttempt(storage, events, logger, stepRecord, context, step, result, definition) {
|
|
385
|
+
await markStepSucceeded(storage, stepRecord.id, result);
|
|
386
|
+
if (definition.hooks?.afterStep) {
|
|
387
|
+
await definition.hooks.afterStep(context, step, result);
|
|
388
|
+
}
|
|
389
|
+
safeEmitEvent(events, logger, {
|
|
390
|
+
runId: context.runId,
|
|
391
|
+
kind: context.kind,
|
|
392
|
+
eventType: "step.completed",
|
|
393
|
+
stepKey: step.key,
|
|
394
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
395
|
+
payload: { result }
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
function toError(error) {
|
|
399
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
400
|
+
}
|
|
401
|
+
async function runOnStepErrorHook(definition, context, step, lastError, logger) {
|
|
402
|
+
if (!definition.hooks?.onStepError) return;
|
|
403
|
+
try {
|
|
404
|
+
await definition.hooks.onStepError(context, step, lastError);
|
|
405
|
+
} catch (hookError) {
|
|
406
|
+
logger.error("onStepError hook failed for run " + context.runId + ":", hookError);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
function emitStepFailedEvent(events, logger, context, step, lastError, attempt) {
|
|
410
|
+
safeEmitEvent(events, logger, {
|
|
411
|
+
runId: context.runId,
|
|
412
|
+
kind: context.kind,
|
|
413
|
+
eventType: "step.failed",
|
|
414
|
+
stepKey: step.key,
|
|
415
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
416
|
+
payload: { error: lastError.message, attempt }
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
function handleSkipAction(events, logger, context, step, lastError) {
|
|
420
|
+
logger.warn(`Step "${step.key}" failed, skipping (strategy: skip)`);
|
|
421
|
+
safeEmitEvent(events, logger, {
|
|
422
|
+
runId: context.runId,
|
|
423
|
+
kind: context.kind,
|
|
424
|
+
eventType: "step.skipped",
|
|
425
|
+
stepKey: step.key,
|
|
426
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
427
|
+
payload: { reason: "error", error: lastError.message }
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
async function handleRetryAction(storage, events, logger, abortController, context, step, attempt, policy, lastError, stepRecordId) {
|
|
431
|
+
const delay = calculateRetryDelay(attempt, policy.retryDelay, policy.retryBackoff);
|
|
432
|
+
logger.warn(
|
|
433
|
+
`Step "${step.key}" failed, retrying in ${delay}ms (attempt ${attempt}/${policy.maxRetries})`
|
|
434
|
+
);
|
|
435
|
+
safeEmitEvent(events, logger, {
|
|
436
|
+
runId: context.runId,
|
|
437
|
+
kind: context.kind,
|
|
438
|
+
eventType: "step.retry",
|
|
439
|
+
stepKey: step.key,
|
|
440
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
441
|
+
payload: { attempt, maxRetries: policy.maxRetries, delay, error: lastError.message }
|
|
442
|
+
});
|
|
443
|
+
try {
|
|
444
|
+
await sleep(delay, abortController.signal);
|
|
445
|
+
} catch (sleepError) {
|
|
446
|
+
if (sleepError instanceof WorkflowCanceledError) {
|
|
447
|
+
await markStepCanceled(storage, stepRecordId);
|
|
448
|
+
}
|
|
449
|
+
throw sleepError;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
async function executeWithTimeout(fn, timeoutMs, signal, stepKey) {
|
|
453
|
+
let timeoutId;
|
|
454
|
+
let onAbort;
|
|
455
|
+
try {
|
|
456
|
+
return await Promise.race([
|
|
457
|
+
fn(),
|
|
458
|
+
new Promise((_, reject) => {
|
|
459
|
+
timeoutId = setTimeout(() => {
|
|
460
|
+
reject(new StepTimeoutError(stepKey, timeoutMs));
|
|
461
|
+
}, timeoutMs);
|
|
462
|
+
onAbort = () => {
|
|
463
|
+
clearTimeout(timeoutId);
|
|
464
|
+
reject(new WorkflowCanceledError("run"));
|
|
465
|
+
};
|
|
466
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
467
|
+
})
|
|
468
|
+
]);
|
|
469
|
+
} finally {
|
|
470
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
471
|
+
if (onAbort) signal.removeEventListener("abort", onAbort);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
async function raceWithAbort(promise, signal) {
|
|
475
|
+
if (signal.aborted) {
|
|
476
|
+
throw new WorkflowCanceledError("run");
|
|
477
|
+
}
|
|
478
|
+
let onAbort;
|
|
479
|
+
try {
|
|
480
|
+
return await Promise.race([
|
|
481
|
+
promise,
|
|
482
|
+
new Promise((_, reject) => {
|
|
483
|
+
onAbort = () => reject(new WorkflowCanceledError("run"));
|
|
484
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
485
|
+
})
|
|
486
|
+
]);
|
|
487
|
+
} finally {
|
|
488
|
+
if (onAbort) signal.removeEventListener("abort", onAbort);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// src/core/orchestrator/plan-resolution.ts
|
|
493
|
+
function buildContext(options) {
|
|
213
494
|
const {
|
|
214
495
|
runId,
|
|
215
496
|
definition,
|
|
216
497
|
input,
|
|
217
|
-
metadata
|
|
218
|
-
storage,
|
|
498
|
+
metadata,
|
|
219
499
|
events,
|
|
220
500
|
logger,
|
|
221
501
|
abortController,
|
|
222
502
|
spawnChild,
|
|
223
|
-
|
|
503
|
+
checkpointResults
|
|
224
504
|
} = options;
|
|
225
|
-
|
|
226
|
-
const isResume = !!checkpoint;
|
|
227
|
-
await storage.updateRun(runId, {
|
|
228
|
-
status: "running",
|
|
229
|
-
...isResume ? {} : { startedAt: /* @__PURE__ */ new Date() }
|
|
230
|
-
});
|
|
231
|
-
emitEvent(events, logger, {
|
|
232
|
-
runId,
|
|
233
|
-
kind: definition.kind,
|
|
234
|
-
eventType: isResume ? "run.resumed" : "run.started",
|
|
235
|
-
timestamp: /* @__PURE__ */ new Date(),
|
|
236
|
-
payload: isResume ? { resumedFrom: Array.from(checkpoint.completedStepKeys) } : void 0
|
|
237
|
-
});
|
|
238
|
-
const context = {
|
|
505
|
+
return {
|
|
239
506
|
runId,
|
|
240
507
|
stepId: "",
|
|
241
|
-
//
|
|
508
|
+
// set per-step by the step-runner before each handler call
|
|
242
509
|
kind: definition.kind,
|
|
243
510
|
input,
|
|
244
|
-
results:
|
|
511
|
+
results: checkpointResults ? { ...checkpointResults } : {},
|
|
245
512
|
metadata,
|
|
246
513
|
logger: createScopedLogger(logger, runId),
|
|
247
514
|
signal: abortController.signal,
|
|
248
515
|
spawnChild,
|
|
249
516
|
emit: (eventType, payload) => {
|
|
250
|
-
|
|
517
|
+
safeEmitEvent(events, logger, {
|
|
251
518
|
runId,
|
|
252
519
|
kind: definition.kind,
|
|
253
520
|
eventType,
|
|
@@ -256,17 +523,66 @@ async function executeWorkflow(options) {
|
|
|
256
523
|
});
|
|
257
524
|
}
|
|
258
525
|
};
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
if (definition.timeout) {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
abortController.abort();
|
|
268
|
-
}, definition.timeout);
|
|
526
|
+
}
|
|
527
|
+
function setupWorkflowTimeout(runId, definition, abortController) {
|
|
528
|
+
if (!definition.timeout) {
|
|
529
|
+
return {
|
|
530
|
+
clear: () => {
|
|
531
|
+
},
|
|
532
|
+
getTimeoutError: () => void 0
|
|
533
|
+
};
|
|
269
534
|
}
|
|
535
|
+
let timeoutError;
|
|
536
|
+
const timeoutId = setTimeout(() => {
|
|
537
|
+
timeoutError = new WorkflowTimeoutError(runId, definition.timeout);
|
|
538
|
+
abortController.abort();
|
|
539
|
+
}, definition.timeout);
|
|
540
|
+
return {
|
|
541
|
+
clear: () => clearTimeout(timeoutId),
|
|
542
|
+
getTimeoutError: () => timeoutError
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
function initialCompletedSteps(checkpoint) {
|
|
546
|
+
return checkpoint ? Array.from(checkpoint.completedStepKeys) : [];
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// src/core/orchestrator.ts
|
|
550
|
+
async function executeWorkflow(options) {
|
|
551
|
+
const {
|
|
552
|
+
runId,
|
|
553
|
+
definition,
|
|
554
|
+
input,
|
|
555
|
+
metadata = {},
|
|
556
|
+
storage,
|
|
557
|
+
events,
|
|
558
|
+
logger,
|
|
559
|
+
abortController,
|
|
560
|
+
spawnChild,
|
|
561
|
+
checkpoint
|
|
562
|
+
} = options;
|
|
563
|
+
const startTime = Date.now();
|
|
564
|
+
const isResume = !!checkpoint;
|
|
565
|
+
await markRunRunning(storage, runId, isResume);
|
|
566
|
+
safeEmitEvent(events, logger, {
|
|
567
|
+
runId,
|
|
568
|
+
kind: definition.kind,
|
|
569
|
+
eventType: isResume ? "run.resumed" : "run.started",
|
|
570
|
+
timestamp: /* @__PURE__ */ new Date(),
|
|
571
|
+
payload: isResume ? { resumedFrom: Array.from(checkpoint.completedStepKeys) } : void 0
|
|
572
|
+
});
|
|
573
|
+
const completedSteps = initialCompletedSteps(checkpoint);
|
|
574
|
+
const context = buildContext({
|
|
575
|
+
runId,
|
|
576
|
+
definition,
|
|
577
|
+
input,
|
|
578
|
+
metadata,
|
|
579
|
+
events,
|
|
580
|
+
logger,
|
|
581
|
+
abortController,
|
|
582
|
+
spawnChild,
|
|
583
|
+
checkpointResults: checkpoint?.results
|
|
584
|
+
});
|
|
585
|
+
const timeoutHandle = setupWorkflowTimeout(runId, definition, abortController);
|
|
270
586
|
try {
|
|
271
587
|
if (definition.hooks?.beforeRun) {
|
|
272
588
|
await definition.hooks.beforeRun(context);
|
|
@@ -277,10 +593,8 @@ async function executeWorkflow(options) {
|
|
|
277
593
|
}
|
|
278
594
|
context.currentStep = step.key;
|
|
279
595
|
if (checkpoint?.completedStepKeys.has(step.key)) {
|
|
280
|
-
logger.info(
|
|
281
|
-
|
|
282
|
-
);
|
|
283
|
-
emitEvent(events, logger, {
|
|
596
|
+
logger.info(`Skipping step "${step.key}" (already completed in previous run)`);
|
|
597
|
+
safeEmitEvent(events, logger, {
|
|
284
598
|
runId,
|
|
285
599
|
kind: definition.kind,
|
|
286
600
|
eventType: "step.skipped",
|
|
@@ -294,7 +608,7 @@ async function executeWorkflow(options) {
|
|
|
294
608
|
const shouldSkip = await step.skipIf(context);
|
|
295
609
|
if (shouldSkip) {
|
|
296
610
|
logger.info(`Skipping step "${step.key}" (condition met)`);
|
|
297
|
-
|
|
611
|
+
safeEmitEvent(events, logger, {
|
|
298
612
|
runId,
|
|
299
613
|
kind: definition.kind,
|
|
300
614
|
eventType: "step.skipped",
|
|
@@ -312,13 +626,10 @@ async function executeWorkflow(options) {
|
|
|
312
626
|
abortController
|
|
313
627
|
});
|
|
314
628
|
context.results[step.key] = result;
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
});
|
|
318
|
-
}
|
|
319
|
-
if (workflowTimeoutId) {
|
|
320
|
-
clearTimeout(workflowTimeoutId);
|
|
629
|
+
completedSteps.push(step.key);
|
|
630
|
+
await saveCheckpoint(storage, runId, context.results, completedSteps);
|
|
321
631
|
}
|
|
632
|
+
timeoutHandle.clear();
|
|
322
633
|
const duration = Date.now() - startTime;
|
|
323
634
|
const runResult = {
|
|
324
635
|
status: "succeeded",
|
|
@@ -328,12 +639,8 @@ async function executeWorkflow(options) {
|
|
|
328
639
|
if (definition.hooks?.afterRun) {
|
|
329
640
|
await definition.hooks.afterRun(context, runResult);
|
|
330
641
|
}
|
|
331
|
-
await storage
|
|
332
|
-
|
|
333
|
-
context: context.results,
|
|
334
|
-
finishedAt: /* @__PURE__ */ new Date()
|
|
335
|
-
});
|
|
336
|
-
emitEvent(events, logger, {
|
|
642
|
+
await markRunSucceeded(storage, runId, context.results);
|
|
643
|
+
safeEmitEvent(events, logger, {
|
|
337
644
|
runId,
|
|
338
645
|
kind: definition.kind,
|
|
339
646
|
eventType: "run.completed",
|
|
@@ -342,16 +649,11 @@ async function executeWorkflow(options) {
|
|
|
342
649
|
});
|
|
343
650
|
return runResult;
|
|
344
651
|
} catch (error) {
|
|
345
|
-
|
|
346
|
-
clearTimeout(workflowTimeoutId);
|
|
347
|
-
}
|
|
652
|
+
timeoutHandle.clear();
|
|
348
653
|
const duration = Date.now() - startTime;
|
|
349
|
-
const actualError =
|
|
654
|
+
const actualError = timeoutHandle.getTimeoutError() ?? error;
|
|
350
655
|
const workflowError = WorkflowEngineError.fromError(actualError);
|
|
351
|
-
const
|
|
352
|
-
const isCanceled = !isTimeout && actualError instanceof WorkflowCanceledError;
|
|
353
|
-
const status = isCanceled ? "canceled" : "failed";
|
|
354
|
-
const eventType = isTimeout ? "run.timeout" : isCanceled ? "run.canceled" : "run.failed";
|
|
656
|
+
const { status, eventType } = decideRunFinalState(actualError);
|
|
355
657
|
const runResult = {
|
|
356
658
|
status,
|
|
357
659
|
results: context.results,
|
|
@@ -362,16 +664,11 @@ async function executeWorkflow(options) {
|
|
|
362
664
|
try {
|
|
363
665
|
await definition.hooks.afterRun(context, runResult);
|
|
364
666
|
} catch (hookError) {
|
|
365
|
-
logger.error("afterRun hook failed:", hookError);
|
|
667
|
+
logger.error("afterRun hook failed for run " + runId + ":", hookError);
|
|
366
668
|
}
|
|
367
669
|
}
|
|
368
|
-
await storage
|
|
369
|
-
|
|
370
|
-
context: context.results,
|
|
371
|
-
error: workflowError,
|
|
372
|
-
finishedAt: /* @__PURE__ */ new Date()
|
|
373
|
-
});
|
|
374
|
-
emitEvent(events, logger, {
|
|
670
|
+
await markRunFinal(storage, runId, status, context.results, workflowError);
|
|
671
|
+
safeEmitEvent(events, logger, {
|
|
375
672
|
runId,
|
|
376
673
|
kind: definition.kind,
|
|
377
674
|
eventType,
|
|
@@ -381,193 +678,272 @@ async function executeWorkflow(options) {
|
|
|
381
678
|
return runResult;
|
|
382
679
|
}
|
|
383
680
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
startedAt: /* @__PURE__ */ new Date()
|
|
404
|
-
});
|
|
405
|
-
context.stepId = stepRecord.id;
|
|
406
|
-
emitEvent(events, logger, {
|
|
407
|
-
runId: context.runId,
|
|
408
|
-
kind: context.kind,
|
|
409
|
-
eventType: "step.started",
|
|
410
|
-
stepKey: step.key,
|
|
411
|
-
timestamp: /* @__PURE__ */ new Date(),
|
|
412
|
-
payload: { attempt }
|
|
413
|
-
});
|
|
414
|
-
if (definition.hooks?.beforeStep) {
|
|
415
|
-
await definition.hooks.beforeStep(context, step);
|
|
681
|
+
|
|
682
|
+
// src/core/engine/concurrency.ts
|
|
683
|
+
var ConcurrencyManager = class {
|
|
684
|
+
activeRuns = /* @__PURE__ */ new Map();
|
|
685
|
+
runQueue = [];
|
|
686
|
+
timerHandles = /* @__PURE__ */ new Set();
|
|
687
|
+
storage;
|
|
688
|
+
events;
|
|
689
|
+
logger;
|
|
690
|
+
maxConcurrency;
|
|
691
|
+
constructor(deps) {
|
|
692
|
+
this.storage = deps.storage;
|
|
693
|
+
this.events = deps.events;
|
|
694
|
+
this.logger = deps.logger;
|
|
695
|
+
this.maxConcurrency = deps.maxConcurrency;
|
|
696
|
+
}
|
|
697
|
+
hasCapacity() {
|
|
698
|
+
if (this.maxConcurrency === void 0 || this.maxConcurrency <= 0) {
|
|
699
|
+
return true;
|
|
416
700
|
}
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
701
|
+
return this.activeRuns.size < this.maxConcurrency;
|
|
702
|
+
}
|
|
703
|
+
getActiveCount() {
|
|
704
|
+
return this.activeRuns.size;
|
|
705
|
+
}
|
|
706
|
+
getQueuedCount() {
|
|
707
|
+
return this.runQueue.length;
|
|
708
|
+
}
|
|
709
|
+
isActive(runId) {
|
|
710
|
+
return this.activeRuns.has(runId);
|
|
711
|
+
}
|
|
712
|
+
/** Insert a run in priority order (higher priority first, FIFO within same). */
|
|
713
|
+
enqueue(run) {
|
|
714
|
+
let inserted = false;
|
|
715
|
+
for (let i = 0; i < this.runQueue.length; i++) {
|
|
716
|
+
if (run.priority > this.runQueue[i].priority) {
|
|
717
|
+
this.runQueue.splice(i, 0, run);
|
|
718
|
+
inserted = true;
|
|
719
|
+
break;
|
|
431
720
|
}
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
721
|
+
}
|
|
722
|
+
if (!inserted) {
|
|
723
|
+
this.runQueue.push(run);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
/** Remove a run from the queue if present. Returns true if it was queued. */
|
|
727
|
+
removeFromQueue(runId) {
|
|
728
|
+
const idx = this.runQueue.findIndex((q) => q.runId === runId);
|
|
729
|
+
if (idx === -1) return false;
|
|
730
|
+
this.runQueue.splice(idx, 1);
|
|
731
|
+
return true;
|
|
732
|
+
}
|
|
733
|
+
registerActive(runId, controller) {
|
|
734
|
+
this.activeRuns.set(runId, controller);
|
|
735
|
+
}
|
|
736
|
+
unregisterActive(runId) {
|
|
737
|
+
this.activeRuns.delete(runId);
|
|
738
|
+
}
|
|
739
|
+
abortActive(runId) {
|
|
740
|
+
const controller = this.activeRuns.get(runId);
|
|
741
|
+
if (!controller) return false;
|
|
742
|
+
controller.abort();
|
|
743
|
+
return true;
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* Abort every in-flight run. `onAbort` is called for each run id before its
|
|
747
|
+
* controller is signaled (so callers can log, emit, etc.).
|
|
748
|
+
*/
|
|
749
|
+
abortAllActive(onAbort) {
|
|
750
|
+
for (const [runId, controller] of this.activeRuns) {
|
|
751
|
+
if (onAbort) onAbort(runId);
|
|
752
|
+
controller.abort();
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
/** Forget all active-run bookkeeping without aborting (used during shutdown). */
|
|
756
|
+
clearActive() {
|
|
757
|
+
this.activeRuns.clear();
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* Schedule a launch via setTimeout (when `delay` is truthy) or setImmediate
|
|
761
|
+
* (otherwise). Tracks the handle so it can be cleared during shutdown.
|
|
762
|
+
*/
|
|
763
|
+
scheduleLaunch(fn, delay) {
|
|
764
|
+
if (delay) {
|
|
765
|
+
const handle = setTimeout(fn, delay);
|
|
766
|
+
this.timerHandles.add(handle);
|
|
767
|
+
} else {
|
|
768
|
+
const handle = setImmediate(fn);
|
|
769
|
+
this.timerHandles.add(handle);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
clearTimers() {
|
|
773
|
+
for (const handle of this.timerHandles) {
|
|
774
|
+
if (typeof handle === "object" && "ref" in handle) {
|
|
775
|
+
clearImmediate(handle);
|
|
776
|
+
} else {
|
|
777
|
+
clearTimeout(handle);
|
|
439
778
|
}
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
timestamp: /* @__PURE__ */ new Date()
|
|
464
|
-
payload: { error: lastError.message, attempt }
|
|
779
|
+
}
|
|
780
|
+
this.timerHandles.clear();
|
|
781
|
+
}
|
|
782
|
+
/**
|
|
783
|
+
* Drain the queue while capacity is available, dispatching each item via
|
|
784
|
+
* `dispatch(run)`. Each iteration is wrapped in try/catch so that a failure
|
|
785
|
+
* dispatching one queued item (e.g., an event-transport that throws) cannot
|
|
786
|
+
* tear down the loop and leave remaining items stuck in the queue. Failed
|
|
787
|
+
* dispatches are logged, the run is marked as failed in storage as
|
|
788
|
+
* best-effort telemetry, and the loop advances to the next item.
|
|
789
|
+
*
|
|
790
|
+
* Emits `run.dequeued` for each dispatched run (this is what the C3
|
|
791
|
+
* resilience test exercises by having the emit throw).
|
|
792
|
+
*/
|
|
793
|
+
processQueue(dispatch) {
|
|
794
|
+
while (this.hasCapacity() && this.runQueue.length > 0) {
|
|
795
|
+
const next = this.runQueue.shift();
|
|
796
|
+
try {
|
|
797
|
+
this.logger.debug(`Starting queued run ${next.runId}`);
|
|
798
|
+
this.events.emit({
|
|
799
|
+
runId: next.runId,
|
|
800
|
+
kind: next.definition.kind,
|
|
801
|
+
eventType: "run.dequeued",
|
|
802
|
+
timestamp: /* @__PURE__ */ new Date()
|
|
465
803
|
});
|
|
466
|
-
|
|
804
|
+
dispatch(next);
|
|
805
|
+
} catch (err) {
|
|
806
|
+
this.recordQueueDispatchFailure(next, err);
|
|
467
807
|
}
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* Record a queue-dispatch failure: log it, emit a failure event, and
|
|
812
|
+
* best-effort mark the run as failed in storage. All side-effects are
|
|
813
|
+
* defensively wrapped so this method itself never throws.
|
|
814
|
+
*/
|
|
815
|
+
recordQueueDispatchFailure(queued, err) {
|
|
816
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
817
|
+
this.logger.error(`Failed to dispatch queued run ${queued.runId}`, err);
|
|
818
|
+
try {
|
|
819
|
+
this.events.emit({
|
|
820
|
+
runId: queued.runId,
|
|
821
|
+
kind: queued.definition.kind,
|
|
822
|
+
eventType: "run.failed",
|
|
480
823
|
timestamp: /* @__PURE__ */ new Date(),
|
|
481
|
-
payload: { error:
|
|
824
|
+
payload: { error: { code: "QUEUE_DISPATCH_FAILED", message } }
|
|
482
825
|
});
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
return void 0;
|
|
494
|
-
}
|
|
495
|
-
if (onError === "retry" && attempt <= maxRetries) {
|
|
496
|
-
const delay = calculateRetryDelay(attempt, retryDelay, retryBackoff);
|
|
497
|
-
logger.warn(
|
|
498
|
-
`Step "${step.key}" failed, retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`
|
|
499
|
-
);
|
|
500
|
-
emitEvent(events, logger, {
|
|
501
|
-
runId: context.runId,
|
|
502
|
-
kind: context.kind,
|
|
503
|
-
eventType: "step.retry",
|
|
504
|
-
stepKey: step.key,
|
|
505
|
-
timestamp: /* @__PURE__ */ new Date(),
|
|
506
|
-
payload: { attempt, maxRetries, delay, error: lastError.message }
|
|
507
|
-
});
|
|
508
|
-
try {
|
|
509
|
-
await sleep(delay, abortController.signal);
|
|
510
|
-
} catch (sleepError) {
|
|
511
|
-
if (sleepError instanceof WorkflowCanceledError) {
|
|
512
|
-
await storage.updateStep(stepRecord.id, {
|
|
513
|
-
status: "canceled",
|
|
514
|
-
finishedAt: /* @__PURE__ */ new Date()
|
|
515
|
-
});
|
|
516
|
-
}
|
|
517
|
-
throw sleepError;
|
|
518
|
-
}
|
|
519
|
-
continue;
|
|
520
|
-
}
|
|
521
|
-
throw new StepError(step.key, lastError.message, attempt, lastError);
|
|
522
|
-
}
|
|
826
|
+
} catch (emitErr) {
|
|
827
|
+
this.logger.error(`Failed to emit dispatch-failure event for ${queued.runId}`, emitErr);
|
|
828
|
+
}
|
|
829
|
+
this.storage.updateRun(queued.runId, {
|
|
830
|
+
status: "failed",
|
|
831
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
832
|
+
error: { code: "QUEUE_DISPATCH_FAILED", message }
|
|
833
|
+
}).catch((updateErr) => {
|
|
834
|
+
this.logger.error(`Failed to record dispatch failure for ${queued.runId}`, updateErr);
|
|
835
|
+
});
|
|
523
836
|
}
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
]);
|
|
542
|
-
} finally {
|
|
543
|
-
if (timeoutId) clearTimeout(timeoutId);
|
|
544
|
-
if (onAbort) signal.removeEventListener("abort", onAbort);
|
|
837
|
+
};
|
|
838
|
+
|
|
839
|
+
// src/core/engine/lifecycle.ts
|
|
840
|
+
var LifecycleManager = class {
|
|
841
|
+
storage;
|
|
842
|
+
events;
|
|
843
|
+
logger;
|
|
844
|
+
registry;
|
|
845
|
+
concurrency;
|
|
846
|
+
launchRun;
|
|
847
|
+
constructor(deps) {
|
|
848
|
+
this.storage = deps.storage;
|
|
849
|
+
this.events = deps.events;
|
|
850
|
+
this.logger = deps.logger;
|
|
851
|
+
this.registry = deps.registry;
|
|
852
|
+
this.concurrency = deps.concurrency;
|
|
853
|
+
this.launchRun = deps.launchRun;
|
|
545
854
|
}
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
855
|
+
/**
|
|
856
|
+
* Initialize the engine's storage adapter if it requires it.
|
|
857
|
+
* Safe to call when the adapter has no `initialize` hook.
|
|
858
|
+
*/
|
|
859
|
+
async initialize() {
|
|
860
|
+
if (this.storage.initialize) {
|
|
861
|
+
await this.storage.initialize();
|
|
862
|
+
}
|
|
550
863
|
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
864
|
+
/**
|
|
865
|
+
* Shutdown the engine gracefully:
|
|
866
|
+
* 1. clear pending timer handles (so no further launches fire)
|
|
867
|
+
* 2. abort all in-flight runs
|
|
868
|
+
* 3. close the event transport
|
|
869
|
+
* 4. close the storage adapter
|
|
870
|
+
* 5. drop active-run bookkeeping
|
|
871
|
+
*/
|
|
872
|
+
async shutdown() {
|
|
873
|
+
this.logger.info("Shutting down workflow engine...");
|
|
874
|
+
this.concurrency.clearTimers();
|
|
875
|
+
this.concurrency.abortAllActive((runId) => {
|
|
876
|
+
this.logger.debug(`Canceling active run: ${runId}`);
|
|
877
|
+
});
|
|
878
|
+
if (this.events.close) {
|
|
879
|
+
this.events.close();
|
|
880
|
+
}
|
|
881
|
+
if (this.storage.close) {
|
|
882
|
+
await this.storage.close();
|
|
883
|
+
}
|
|
884
|
+
this.concurrency.clearActive();
|
|
885
|
+
this.logger.info("Workflow engine shutdown complete");
|
|
886
|
+
}
|
|
887
|
+
/**
|
|
888
|
+
* Resume an interrupted workflow run from its checkpoint.
|
|
889
|
+
* The run must be in 'queued' or 'running' status.
|
|
890
|
+
*/
|
|
891
|
+
async resumeRun(runId) {
|
|
892
|
+
const run = await this.storage.getRun(runId);
|
|
893
|
+
if (!run) {
|
|
894
|
+
throw new RunNotFoundError(runId);
|
|
895
|
+
}
|
|
896
|
+
if (!["queued", "running"].includes(run.status)) {
|
|
897
|
+
throw new Error(`Cannot resume run ${runId}: status is "${run.status}"`);
|
|
898
|
+
}
|
|
899
|
+
const definition = this.registry.get(run.kind);
|
|
900
|
+
if (!definition) {
|
|
901
|
+
throw new WorkflowNotFoundError(run.kind);
|
|
902
|
+
}
|
|
903
|
+
if (this.concurrency.isActive(runId)) {
|
|
904
|
+
this.logger.warn(`Run ${runId} is already active, skipping resume`);
|
|
905
|
+
return runId;
|
|
906
|
+
}
|
|
907
|
+
this.logger.info(`Resuming run ${runId} from checkpoint`);
|
|
908
|
+
const completedStepKeys = new Set(run.completedSteps ?? []);
|
|
909
|
+
this.launchRun(runId, definition, run.input, run.metadata, void 0, {
|
|
910
|
+
completedStepKeys,
|
|
911
|
+
results: run.context
|
|
912
|
+
});
|
|
913
|
+
return runId;
|
|
562
914
|
}
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
915
|
+
/**
|
|
916
|
+
* Get all runs that were interrupted and can be resumed.
|
|
917
|
+
* Returns runs with status 'queued' or 'running'.
|
|
918
|
+
*/
|
|
919
|
+
async getResumableRuns() {
|
|
920
|
+
const result = await this.storage.listRuns({
|
|
921
|
+
status: ["queued", "running"]
|
|
922
|
+
});
|
|
923
|
+
return result.items;
|
|
569
924
|
}
|
|
570
|
-
|
|
925
|
+
/**
|
|
926
|
+
* Resume every interrupted run whose workflow kind is currently registered.
|
|
927
|
+
* Useful for recovering after a server restart.
|
|
928
|
+
*/
|
|
929
|
+
async resumeAllInterrupted() {
|
|
930
|
+
const resumableRuns = await this.getResumableRuns();
|
|
931
|
+
const resumedIds = [];
|
|
932
|
+
for (const run of resumableRuns) {
|
|
933
|
+
if (this.registry.has(run.kind)) {
|
|
934
|
+
try {
|
|
935
|
+
await this.resumeRun(run.id);
|
|
936
|
+
resumedIds.push(run.id);
|
|
937
|
+
} catch (error) {
|
|
938
|
+
this.logger.error(`Failed to resume run ${run.id}:`, error);
|
|
939
|
+
}
|
|
940
|
+
} else {
|
|
941
|
+
this.logger.warn(`Cannot resume run ${run.id}: workflow "${run.kind}" not registered`);
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
return resumedIds;
|
|
945
|
+
}
|
|
946
|
+
};
|
|
571
947
|
|
|
572
948
|
// src/core/engine.ts
|
|
573
949
|
var WorkflowEngine = class _WorkflowEngine {
|
|
@@ -575,14 +951,28 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
575
951
|
storage;
|
|
576
952
|
events;
|
|
577
953
|
logger;
|
|
578
|
-
activeRuns = /* @__PURE__ */ new Map();
|
|
579
954
|
settings;
|
|
580
|
-
|
|
955
|
+
concurrency;
|
|
956
|
+
lifecycle;
|
|
581
957
|
constructor(config = {}) {
|
|
582
958
|
this.storage = config.storage ?? new MemoryStorageAdapter();
|
|
583
959
|
this.events = config.events ?? new MemoryEventTransport();
|
|
584
960
|
this.logger = config.logger ?? new ConsoleLogger();
|
|
585
961
|
this.settings = config.settings ?? {};
|
|
962
|
+
this.concurrency = new ConcurrencyManager({
|
|
963
|
+
storage: this.storage,
|
|
964
|
+
events: this.events,
|
|
965
|
+
logger: this.logger,
|
|
966
|
+
maxConcurrency: this.settings.maxConcurrency
|
|
967
|
+
});
|
|
968
|
+
this.lifecycle = new LifecycleManager({
|
|
969
|
+
storage: this.storage,
|
|
970
|
+
events: this.events,
|
|
971
|
+
logger: this.logger,
|
|
972
|
+
registry: this.registry,
|
|
973
|
+
concurrency: this.concurrency,
|
|
974
|
+
launchRun: (runId, def, input, metadata, delay, checkpoint) => this.launchRun(runId, def, input, metadata, delay, checkpoint)
|
|
975
|
+
});
|
|
586
976
|
}
|
|
587
977
|
/**
|
|
588
978
|
* Initialize the engine and its storage/event adapters.
|
|
@@ -590,31 +980,19 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
590
980
|
* (e.g., PostgresStorageAdapter).
|
|
591
981
|
*/
|
|
592
982
|
async initialize() {
|
|
593
|
-
|
|
594
|
-
await this.storage.initialize();
|
|
595
|
-
}
|
|
983
|
+
return this.lifecycle.initialize();
|
|
596
984
|
}
|
|
597
985
|
/**
|
|
598
986
|
* Get the current number of active runs.
|
|
599
987
|
*/
|
|
600
988
|
getActiveRunCount() {
|
|
601
|
-
return this.
|
|
989
|
+
return this.concurrency.getActiveCount();
|
|
602
990
|
}
|
|
603
991
|
/**
|
|
604
992
|
* Get the number of queued runs waiting for capacity.
|
|
605
993
|
*/
|
|
606
994
|
getQueuedRunCount() {
|
|
607
|
-
return this.
|
|
608
|
-
}
|
|
609
|
-
/**
|
|
610
|
-
* Check if capacity is available for a new run.
|
|
611
|
-
*/
|
|
612
|
-
hasCapacity() {
|
|
613
|
-
const maxConcurrency = this.settings.maxConcurrency;
|
|
614
|
-
if (maxConcurrency === void 0 || maxConcurrency <= 0) {
|
|
615
|
-
return true;
|
|
616
|
-
}
|
|
617
|
-
return this.activeRuns.size < maxConcurrency;
|
|
995
|
+
return this.concurrency.getQueuedCount();
|
|
618
996
|
}
|
|
619
997
|
// ============================================================================
|
|
620
998
|
// Workflow Registration
|
|
@@ -692,13 +1070,9 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
692
1070
|
kind: options.kind,
|
|
693
1071
|
eventType: "run.created",
|
|
694
1072
|
timestamp: /* @__PURE__ */ new Date(),
|
|
695
|
-
payload: {
|
|
696
|
-
input: options.input,
|
|
697
|
-
metadata: options.metadata,
|
|
698
|
-
priority: options.priority ?? 0
|
|
699
|
-
}
|
|
1073
|
+
payload: { input: options.input, metadata: options.metadata, priority: options.priority ?? 0 }
|
|
700
1074
|
});
|
|
701
|
-
if (this.hasCapacity()) {
|
|
1075
|
+
if (this.concurrency.hasCapacity()) {
|
|
702
1076
|
this.executeRun(
|
|
703
1077
|
runId,
|
|
704
1078
|
definition,
|
|
@@ -707,7 +1081,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
707
1081
|
options.delay
|
|
708
1082
|
);
|
|
709
1083
|
} else {
|
|
710
|
-
this.
|
|
1084
|
+
this.concurrency.enqueue({
|
|
711
1085
|
runId,
|
|
712
1086
|
definition,
|
|
713
1087
|
input: options.input ?? {},
|
|
@@ -715,35 +1089,17 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
715
1089
|
priority: options.priority ?? 0,
|
|
716
1090
|
queuedAt: /* @__PURE__ */ new Date()
|
|
717
1091
|
});
|
|
718
|
-
this.logger.debug(
|
|
719
|
-
`Run ${runId} queued (${this.runQueue.length} in queue)`
|
|
720
|
-
);
|
|
1092
|
+
this.logger.debug(`Run ${runId} queued (${this.concurrency.getQueuedCount()} in queue)`);
|
|
721
1093
|
this.events.emit({
|
|
722
1094
|
runId,
|
|
723
1095
|
kind: options.kind,
|
|
724
1096
|
eventType: "run.queued",
|
|
725
1097
|
timestamp: /* @__PURE__ */ new Date(),
|
|
726
|
-
payload: { queuePosition: this.
|
|
1098
|
+
payload: { queuePosition: this.concurrency.getQueuedCount() }
|
|
727
1099
|
});
|
|
728
1100
|
}
|
|
729
1101
|
return runId;
|
|
730
1102
|
}
|
|
731
|
-
/**
|
|
732
|
-
* Queue a run in priority order.
|
|
733
|
-
*/
|
|
734
|
-
queueRun(queuedRun) {
|
|
735
|
-
let inserted = false;
|
|
736
|
-
for (let i = 0; i < this.runQueue.length; i++) {
|
|
737
|
-
if (queuedRun.priority > this.runQueue[i].priority) {
|
|
738
|
-
this.runQueue.splice(i, 0, queuedRun);
|
|
739
|
-
inserted = true;
|
|
740
|
-
break;
|
|
741
|
-
}
|
|
742
|
-
}
|
|
743
|
-
if (!inserted) {
|
|
744
|
-
this.runQueue.push(queuedRun);
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
1103
|
/**
|
|
748
1104
|
* Execute a run (internal method).
|
|
749
1105
|
*/
|
|
@@ -756,7 +1112,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
756
1112
|
*/
|
|
757
1113
|
launchRun(runId, definition, input, metadata, delay, checkpoint) {
|
|
758
1114
|
const abortController = new AbortController();
|
|
759
|
-
this.
|
|
1115
|
+
this.concurrency.registerActive(runId, abortController);
|
|
760
1116
|
const execute = async () => {
|
|
761
1117
|
try {
|
|
762
1118
|
await executeWorkflow({
|
|
@@ -772,31 +1128,26 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
772
1128
|
checkpoint
|
|
773
1129
|
});
|
|
774
1130
|
} finally {
|
|
775
|
-
this.
|
|
1131
|
+
this.concurrency.unregisterActive(runId);
|
|
776
1132
|
this.processQueue();
|
|
777
1133
|
}
|
|
778
1134
|
};
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
1135
|
+
const guarded = () => {
|
|
1136
|
+
execute().catch((err) => {
|
|
1137
|
+
this.concurrency.unregisterActive(runId);
|
|
1138
|
+
this.logger.error(`Unhandled error launching run ${runId}`, err);
|
|
1139
|
+
});
|
|
1140
|
+
};
|
|
1141
|
+
this.concurrency.scheduleLaunch(guarded, delay);
|
|
784
1142
|
}
|
|
785
1143
|
/**
|
|
786
|
-
*
|
|
1144
|
+
* Drain the queue, dispatching each ready run via `executeRun`. Resilience
|
|
1145
|
+
* against per-item dispatch failures lives in `ConcurrencyManager.processQueue`.
|
|
787
1146
|
*/
|
|
788
1147
|
processQueue() {
|
|
789
|
-
|
|
790
|
-
const next = this.runQueue.shift();
|
|
791
|
-
this.logger.debug(`Starting queued run ${next.runId}`);
|
|
792
|
-
this.events.emit({
|
|
793
|
-
runId: next.runId,
|
|
794
|
-
kind: next.definition.kind,
|
|
795
|
-
eventType: "run.dequeued",
|
|
796
|
-
timestamp: /* @__PURE__ */ new Date()
|
|
797
|
-
});
|
|
1148
|
+
this.concurrency.processQueue((next) => {
|
|
798
1149
|
this.executeRun(next.runId, next.definition, next.input, next.metadata);
|
|
799
|
-
}
|
|
1150
|
+
});
|
|
800
1151
|
}
|
|
801
1152
|
/**
|
|
802
1153
|
* Start a child workflow from within a parent workflow.
|
|
@@ -825,14 +1176,8 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
825
1176
|
if (["succeeded", "failed", "canceled", "timeout"].includes(run.status)) {
|
|
826
1177
|
return;
|
|
827
1178
|
}
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
this.runQueue.splice(queueIndex, 1);
|
|
831
|
-
}
|
|
832
|
-
const controller = this.activeRuns.get(runId);
|
|
833
|
-
if (controller) {
|
|
834
|
-
controller.abort();
|
|
835
|
-
}
|
|
1179
|
+
this.concurrency.removeFromQueue(runId);
|
|
1180
|
+
this.concurrency.abortActive(runId);
|
|
836
1181
|
await this.storage.updateRun(runId, {
|
|
837
1182
|
status: "canceled",
|
|
838
1183
|
finishedAt: /* @__PURE__ */ new Date()
|
|
@@ -853,18 +1198,8 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
853
1198
|
async getRunStatus(runId) {
|
|
854
1199
|
return this.storage.getRun(runId);
|
|
855
1200
|
}
|
|
856
|
-
static TERMINAL_STATUSES = [
|
|
857
|
-
|
|
858
|
-
"failed",
|
|
859
|
-
"canceled",
|
|
860
|
-
"timeout"
|
|
861
|
-
];
|
|
862
|
-
static TERMINAL_EVENT_TYPES = [
|
|
863
|
-
"run.completed",
|
|
864
|
-
"run.failed",
|
|
865
|
-
"run.canceled",
|
|
866
|
-
"run.timeout"
|
|
867
|
-
];
|
|
1201
|
+
static TERMINAL_STATUSES = ["succeeded", "failed", "canceled", "timeout"];
|
|
1202
|
+
static TERMINAL_EVENT_TYPES = ["run.completed", "run.failed", "run.canceled", "run.timeout"];
|
|
868
1203
|
/**
|
|
869
1204
|
* Wait for a run to complete.
|
|
870
1205
|
* Subscribes to run events and resolves when a terminal event fires.
|
|
@@ -876,39 +1211,52 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
876
1211
|
*/
|
|
877
1212
|
async waitForRun(runId, options = {}) {
|
|
878
1213
|
const timeout = options.timeout ?? 6e4;
|
|
879
|
-
|
|
880
|
-
if (!existingRun) {
|
|
881
|
-
throw new RunNotFoundError(runId);
|
|
882
|
-
}
|
|
883
|
-
if (_WorkflowEngine.TERMINAL_STATUSES.includes(existingRun.status)) {
|
|
884
|
-
return existingRun;
|
|
885
|
-
}
|
|
886
|
-
return new Promise((resolve, reject) => {
|
|
1214
|
+
return new Promise(async (resolve, reject) => {
|
|
887
1215
|
let timeoutId;
|
|
888
1216
|
let unsubscribe;
|
|
1217
|
+
let settled = false;
|
|
889
1218
|
const cleanup = () => {
|
|
890
1219
|
if (timeoutId) clearTimeout(timeoutId);
|
|
891
1220
|
if (unsubscribe) unsubscribe();
|
|
892
1221
|
};
|
|
893
|
-
|
|
1222
|
+
const settle = (fn) => {
|
|
1223
|
+
if (settled) return;
|
|
1224
|
+
settled = true;
|
|
894
1225
|
cleanup();
|
|
895
|
-
|
|
1226
|
+
fn();
|
|
1227
|
+
};
|
|
1228
|
+
timeoutId = setTimeout(() => {
|
|
1229
|
+
settle(() => reject(new WaitForRunTimeoutError(runId, timeout)));
|
|
896
1230
|
}, timeout);
|
|
897
1231
|
unsubscribe = this.events.subscribe(runId, async (event) => {
|
|
898
1232
|
if (_WorkflowEngine.TERMINAL_EVENT_TYPES.includes(event.eventType)) {
|
|
899
|
-
cleanup();
|
|
900
1233
|
try {
|
|
901
1234
|
const run = await this.storage.getRun(runId);
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
1235
|
+
settle(() => {
|
|
1236
|
+
if (!run) {
|
|
1237
|
+
reject(new RunNotFoundError(runId));
|
|
1238
|
+
} else {
|
|
1239
|
+
resolve(run);
|
|
1240
|
+
}
|
|
1241
|
+
});
|
|
907
1242
|
} catch (err) {
|
|
908
|
-
reject(err);
|
|
1243
|
+
settle(() => reject(err));
|
|
909
1244
|
}
|
|
910
1245
|
}
|
|
911
1246
|
});
|
|
1247
|
+
try {
|
|
1248
|
+
const existingRun = await this.storage.getRun(runId);
|
|
1249
|
+
if (!existingRun) {
|
|
1250
|
+
settle(() => reject(new RunNotFoundError(runId)));
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
if (_WorkflowEngine.TERMINAL_STATUSES.includes(existingRun.status)) {
|
|
1254
|
+
settle(() => resolve(existingRun));
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
} catch (err) {
|
|
1258
|
+
settle(() => reject(err));
|
|
1259
|
+
}
|
|
912
1260
|
});
|
|
913
1261
|
}
|
|
914
1262
|
// ============================================================================
|
|
@@ -925,38 +1273,14 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
925
1273
|
* @throws {WorkflowNotFoundError} If the workflow kind is not registered
|
|
926
1274
|
*/
|
|
927
1275
|
async resumeRun(runId) {
|
|
928
|
-
|
|
929
|
-
if (!run) {
|
|
930
|
-
throw new RunNotFoundError(runId);
|
|
931
|
-
}
|
|
932
|
-
if (!["queued", "running"].includes(run.status)) {
|
|
933
|
-
throw new Error(`Cannot resume run ${runId}: status is "${run.status}"`);
|
|
934
|
-
}
|
|
935
|
-
const definition = this.registry.get(run.kind);
|
|
936
|
-
if (!definition) {
|
|
937
|
-
throw new WorkflowNotFoundError(run.kind);
|
|
938
|
-
}
|
|
939
|
-
if (this.activeRuns.has(runId)) {
|
|
940
|
-
this.logger.warn(`Run ${runId} is already active, skipping resume`);
|
|
941
|
-
return runId;
|
|
942
|
-
}
|
|
943
|
-
this.logger.info(`Resuming run ${runId} from checkpoint`);
|
|
944
|
-
const completedStepKeys = new Set(Object.keys(run.context));
|
|
945
|
-
this.launchRun(runId, definition, run.input, run.metadata, void 0, {
|
|
946
|
-
completedStepKeys,
|
|
947
|
-
results: run.context
|
|
948
|
-
});
|
|
949
|
-
return runId;
|
|
1276
|
+
return this.lifecycle.resumeRun(runId);
|
|
950
1277
|
}
|
|
951
1278
|
/**
|
|
952
1279
|
* Get all runs that were interrupted and can be resumed.
|
|
953
1280
|
* Returns runs with status 'queued' or 'running'.
|
|
954
1281
|
*/
|
|
955
1282
|
async getResumableRuns() {
|
|
956
|
-
|
|
957
|
-
status: ["queued", "running"]
|
|
958
|
-
});
|
|
959
|
-
return result.items;
|
|
1283
|
+
return this.lifecycle.getResumableRuns();
|
|
960
1284
|
}
|
|
961
1285
|
/**
|
|
962
1286
|
* Resume all interrupted runs.
|
|
@@ -965,23 +1289,7 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
965
1289
|
* @returns Array of resumed run IDs
|
|
966
1290
|
*/
|
|
967
1291
|
async resumeAllInterrupted() {
|
|
968
|
-
|
|
969
|
-
const resumedIds = [];
|
|
970
|
-
for (const run of resumableRuns) {
|
|
971
|
-
if (this.registry.has(run.kind)) {
|
|
972
|
-
try {
|
|
973
|
-
await this.resumeRun(run.id);
|
|
974
|
-
resumedIds.push(run.id);
|
|
975
|
-
} catch (error) {
|
|
976
|
-
this.logger.error(`Failed to resume run ${run.id}:`, error);
|
|
977
|
-
}
|
|
978
|
-
} else {
|
|
979
|
-
this.logger.warn(
|
|
980
|
-
`Cannot resume run ${run.id}: workflow "${run.kind}" not registered`
|
|
981
|
-
);
|
|
982
|
-
}
|
|
983
|
-
}
|
|
984
|
-
return resumedIds;
|
|
1292
|
+
return this.lifecycle.resumeAllInterrupted();
|
|
985
1293
|
}
|
|
986
1294
|
// ============================================================================
|
|
987
1295
|
// Event Subscription
|
|
@@ -1030,24 +1338,65 @@ var WorkflowEngine = class _WorkflowEngine {
|
|
|
1030
1338
|
* Cancels all active runs and closes resources.
|
|
1031
1339
|
*/
|
|
1032
1340
|
async shutdown() {
|
|
1033
|
-
this.
|
|
1034
|
-
for (const [runId, controller] of this.activeRuns) {
|
|
1035
|
-
this.logger.debug(`Canceling active run: ${runId}`);
|
|
1036
|
-
controller.abort();
|
|
1037
|
-
}
|
|
1038
|
-
if (this.events.close) {
|
|
1039
|
-
this.events.close();
|
|
1040
|
-
}
|
|
1041
|
-
if (this.storage.close) {
|
|
1042
|
-
await this.storage.close();
|
|
1043
|
-
}
|
|
1044
|
-
this.activeRuns.clear();
|
|
1045
|
-
this.logger.info("Workflow engine shutdown complete");
|
|
1341
|
+
return this.lifecycle.shutdown();
|
|
1046
1342
|
}
|
|
1047
1343
|
};
|
|
1048
1344
|
|
|
1049
|
-
// src/planning/
|
|
1345
|
+
// src/planning/evaluate-condition.ts
|
|
1050
1346
|
import RE2 from "re2";
|
|
1347
|
+
var MAX_REGEX_PATTERN_LENGTH = 512;
|
|
1348
|
+
function getNestedValue(obj, path) {
|
|
1349
|
+
return path.split(".").reduce((current, key) => {
|
|
1350
|
+
if (current && typeof current === "object" && key in current) {
|
|
1351
|
+
return current[key];
|
|
1352
|
+
}
|
|
1353
|
+
return void 0;
|
|
1354
|
+
}, obj);
|
|
1355
|
+
}
|
|
1356
|
+
function evaluateCondition(operator, fieldValue, conditionValue) {
|
|
1357
|
+
switch (operator) {
|
|
1358
|
+
case "eq":
|
|
1359
|
+
return fieldValue === conditionValue;
|
|
1360
|
+
case "neq":
|
|
1361
|
+
return fieldValue !== conditionValue;
|
|
1362
|
+
case "gt":
|
|
1363
|
+
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue > conditionValue : false;
|
|
1364
|
+
case "gte":
|
|
1365
|
+
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue >= conditionValue : false;
|
|
1366
|
+
case "lt":
|
|
1367
|
+
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue < conditionValue : false;
|
|
1368
|
+
case "lte":
|
|
1369
|
+
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue <= conditionValue : false;
|
|
1370
|
+
case "contains":
|
|
1371
|
+
if (typeof fieldValue === "string" && typeof conditionValue === "string") {
|
|
1372
|
+
return fieldValue.includes(conditionValue);
|
|
1373
|
+
}
|
|
1374
|
+
if (Array.isArray(fieldValue)) {
|
|
1375
|
+
return fieldValue.includes(conditionValue);
|
|
1376
|
+
}
|
|
1377
|
+
return false;
|
|
1378
|
+
case "matches":
|
|
1379
|
+
if (typeof fieldValue === "string" && typeof conditionValue === "string") {
|
|
1380
|
+
if (conditionValue.length > MAX_REGEX_PATTERN_LENGTH) {
|
|
1381
|
+
return false;
|
|
1382
|
+
}
|
|
1383
|
+
try {
|
|
1384
|
+
return new RE2(conditionValue).test(fieldValue);
|
|
1385
|
+
} catch {
|
|
1386
|
+
return false;
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
return false;
|
|
1390
|
+
case "exists":
|
|
1391
|
+
return fieldValue !== void 0 && fieldValue !== null && fieldValue !== "";
|
|
1392
|
+
case "notExists":
|
|
1393
|
+
return fieldValue === void 0 || fieldValue === null || fieldValue === "";
|
|
1394
|
+
default:
|
|
1395
|
+
return false;
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
// src/planning/registry.ts
|
|
1051
1400
|
var MemoryStepHandlerRegistry = class {
|
|
1052
1401
|
handlers = /* @__PURE__ */ new Map();
|
|
1053
1402
|
tagIndex = /* @__PURE__ */ new Map();
|
|
@@ -1218,71 +1567,14 @@ var MemoryRecipeRegistry = class {
|
|
|
1218
1567
|
/**
|
|
1219
1568
|
* Evaluate recipe conditions against an input.
|
|
1220
1569
|
* Returns true if all conditions match.
|
|
1221
|
-
*
|
|
1222
|
-
* Note: This intentionally mirrors the condition evaluation in planner.ts
|
|
1223
|
-
* to keep registry queries independent of the planner implementation.
|
|
1224
1570
|
*/
|
|
1225
1571
|
evaluateConditions(conditions, input) {
|
|
1226
1572
|
if (!conditions || conditions.length === 0) return true;
|
|
1227
1573
|
return conditions.every((condition) => {
|
|
1228
|
-
const fieldValue =
|
|
1229
|
-
return
|
|
1574
|
+
const fieldValue = getNestedValue(input, condition.field);
|
|
1575
|
+
return evaluateCondition(condition.operator, fieldValue, condition.value);
|
|
1230
1576
|
});
|
|
1231
1577
|
}
|
|
1232
|
-
/**
|
|
1233
|
-
* Get a nested value from an object using dot notation.
|
|
1234
|
-
*/
|
|
1235
|
-
getNestedValue(obj, path) {
|
|
1236
|
-
return path.split(".").reduce((current, key) => {
|
|
1237
|
-
if (current && typeof current === "object" && key in current) {
|
|
1238
|
-
return current[key];
|
|
1239
|
-
}
|
|
1240
|
-
return void 0;
|
|
1241
|
-
}, obj);
|
|
1242
|
-
}
|
|
1243
|
-
/**
|
|
1244
|
-
* Evaluate a single condition.
|
|
1245
|
-
*/
|
|
1246
|
-
evaluateCondition(condition, fieldValue) {
|
|
1247
|
-
const { operator, value } = condition;
|
|
1248
|
-
switch (operator) {
|
|
1249
|
-
case "eq":
|
|
1250
|
-
return fieldValue === value;
|
|
1251
|
-
case "neq":
|
|
1252
|
-
return fieldValue !== value;
|
|
1253
|
-
case "gt":
|
|
1254
|
-
return typeof fieldValue === "number" && typeof value === "number" ? fieldValue > value : false;
|
|
1255
|
-
case "gte":
|
|
1256
|
-
return typeof fieldValue === "number" && typeof value === "number" ? fieldValue >= value : false;
|
|
1257
|
-
case "lt":
|
|
1258
|
-
return typeof fieldValue === "number" && typeof value === "number" ? fieldValue < value : false;
|
|
1259
|
-
case "lte":
|
|
1260
|
-
return typeof fieldValue === "number" && typeof value === "number" ? fieldValue <= value : false;
|
|
1261
|
-
case "contains":
|
|
1262
|
-
if (typeof fieldValue === "string" && typeof value === "string") {
|
|
1263
|
-
return fieldValue.includes(value);
|
|
1264
|
-
}
|
|
1265
|
-
if (Array.isArray(fieldValue)) {
|
|
1266
|
-
return fieldValue.includes(value);
|
|
1267
|
-
}
|
|
1268
|
-
return false;
|
|
1269
|
-
case "matches":
|
|
1270
|
-
if (typeof fieldValue === "string" && typeof value === "string") {
|
|
1271
|
-
try {
|
|
1272
|
-
return new RE2(value).test(fieldValue);
|
|
1273
|
-
} catch {
|
|
1274
|
-
return false;
|
|
1275
|
-
}
|
|
1276
|
-
}
|
|
1277
|
-
return false;
|
|
1278
|
-
case "exists":
|
|
1279
|
-
return fieldValue !== void 0 && fieldValue !== null && fieldValue !== "";
|
|
1280
|
-
case "notExists":
|
|
1281
|
-
return fieldValue === void 0 || fieldValue === null || fieldValue === "";
|
|
1282
|
-
default:
|
|
1283
|
-
return false;
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
1578
|
};
|
|
1287
1579
|
function createRegistry() {
|
|
1288
1580
|
return {
|
|
@@ -1292,54 +1584,6 @@ function createRegistry() {
|
|
|
1292
1584
|
}
|
|
1293
1585
|
|
|
1294
1586
|
// src/planning/planner.ts
|
|
1295
|
-
import RE22 from "re2";
|
|
1296
|
-
function getNestedValue(obj, path) {
|
|
1297
|
-
return path.split(".").reduce((current, key) => {
|
|
1298
|
-
if (current && typeof current === "object" && key in current) {
|
|
1299
|
-
return current[key];
|
|
1300
|
-
}
|
|
1301
|
-
return void 0;
|
|
1302
|
-
}, obj);
|
|
1303
|
-
}
|
|
1304
|
-
function evaluateCondition(operator, fieldValue, conditionValue) {
|
|
1305
|
-
switch (operator) {
|
|
1306
|
-
case "eq":
|
|
1307
|
-
return fieldValue === conditionValue;
|
|
1308
|
-
case "neq":
|
|
1309
|
-
return fieldValue !== conditionValue;
|
|
1310
|
-
case "gt":
|
|
1311
|
-
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue > conditionValue : false;
|
|
1312
|
-
case "gte":
|
|
1313
|
-
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue >= conditionValue : false;
|
|
1314
|
-
case "lt":
|
|
1315
|
-
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue < conditionValue : false;
|
|
1316
|
-
case "lte":
|
|
1317
|
-
return typeof fieldValue === "number" && typeof conditionValue === "number" ? fieldValue <= conditionValue : false;
|
|
1318
|
-
case "contains":
|
|
1319
|
-
if (typeof fieldValue === "string" && typeof conditionValue === "string") {
|
|
1320
|
-
return fieldValue.includes(conditionValue);
|
|
1321
|
-
}
|
|
1322
|
-
if (Array.isArray(fieldValue)) {
|
|
1323
|
-
return fieldValue.includes(conditionValue);
|
|
1324
|
-
}
|
|
1325
|
-
return false;
|
|
1326
|
-
case "matches":
|
|
1327
|
-
if (typeof fieldValue === "string" && typeof conditionValue === "string") {
|
|
1328
|
-
try {
|
|
1329
|
-
return new RE22(conditionValue).test(fieldValue);
|
|
1330
|
-
} catch {
|
|
1331
|
-
return false;
|
|
1332
|
-
}
|
|
1333
|
-
}
|
|
1334
|
-
return false;
|
|
1335
|
-
case "exists":
|
|
1336
|
-
return fieldValue !== void 0 && fieldValue !== null && fieldValue !== "";
|
|
1337
|
-
case "notExists":
|
|
1338
|
-
return fieldValue === void 0 || fieldValue === null || fieldValue === "";
|
|
1339
|
-
default:
|
|
1340
|
-
return false;
|
|
1341
|
-
}
|
|
1342
|
-
}
|
|
1343
1587
|
function scoreConditions(conditions, input) {
|
|
1344
1588
|
if (!conditions || conditions.length === 0) {
|
|
1345
1589
|
return 10;
|
|
@@ -1525,9 +1769,7 @@ var RuleBasedPlanner = class {
|
|
|
1525
1769
|
}
|
|
1526
1770
|
if (this.validateHandlers && this.handlerRegistry) {
|
|
1527
1771
|
if (!this.handlerRegistry.has(step.handlerRef)) {
|
|
1528
|
-
errors.push(
|
|
1529
|
-
`Step '${step.key}' references unknown handler: ${step.handlerRef}`
|
|
1530
|
-
);
|
|
1772
|
+
errors.push(`Step '${step.key}' references unknown handler: ${step.handlerRef}`);
|
|
1531
1773
|
}
|
|
1532
1774
|
}
|
|
1533
1775
|
}
|
|
@@ -1548,12 +1790,7 @@ var RuleBasedPlanner = class {
|
|
|
1548
1790
|
* Estimate resources required for a plan.
|
|
1549
1791
|
*/
|
|
1550
1792
|
estimateResources(plan) {
|
|
1551
|
-
const {
|
|
1552
|
-
apiCallsPerStep,
|
|
1553
|
-
tokensPerStep,
|
|
1554
|
-
durationPerStep,
|
|
1555
|
-
apiCallsPerChild
|
|
1556
|
-
} = this.resourceEstimates;
|
|
1793
|
+
const { apiCallsPerStep, tokensPerStep, durationPerStep, apiCallsPerChild } = this.resourceEstimates;
|
|
1557
1794
|
const stepCount = plan.steps.length;
|
|
1558
1795
|
const childCount = plan.childWorkflows?.length ?? 0;
|
|
1559
1796
|
return {
|