@workflow/world-local 5.0.0-beta.3 → 5.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +6 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js.map +1 -1
- package/dist/fs.d.ts +7 -0
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +14 -3
- package/dist/fs.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/index.js.map +1 -1
- package/dist/storage/events-storage.d.ts.map +1 -1
- package/dist/storage/events-storage.js +679 -601
- package/dist/storage/events-storage.js.map +1 -1
- package/dist/storage/index.d.ts +10 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +8 -10
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/runs-storage.d.ts +25 -3
- package/dist/storage/runs-storage.d.ts.map +1 -1
- package/dist/storage/runs-storage.js +3 -1
- package/dist/storage/runs-storage.js.map +1 -1
- package/package.json +4 -4
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowWorldError, } from '@workflow/errors';
|
|
3
|
+
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors';
|
|
4
4
|
import { EventSchema, HookSchema, isLegacySpecVersion, requiresNewerWorld, SPEC_VERSION_CURRENT, StepSchema, validateUlidTimestamp, WaitSchema, WorkflowRunSchema, } from '@workflow/world';
|
|
5
5
|
import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js';
|
|
6
6
|
import { assertSafeEntityId, deleteJSON, jsonReplacer, listJSONFiles, paginatedFileSystemQuery, readJSONWithFallback, resolveWithinBase, taggedPath, writeExclusive, writeJSON, } from '../fs.js';
|
|
@@ -8,6 +8,40 @@ import { stripEventDataRefs } from './filters.js';
|
|
|
8
8
|
import { getObjectCreatedAt, hashToken, monotonicUlid } from './helpers.js';
|
|
9
9
|
import { deleteAllHooksForRun } from './hooks-storage.js';
|
|
10
10
|
import { handleLegacyEvent } from './legacy.js';
|
|
11
|
+
/**
|
|
12
|
+
* Per-step in-process async mutex. Serializes concurrent `events.create` calls
|
|
13
|
+
* that target the same step, so that the "check terminal state, then write step
|
|
14
|
+
* entity + event" sequence is atomic. Without this, two concurrent step_started
|
|
15
|
+
* calls can both pass the not-terminal check and both write step_started events
|
|
16
|
+
* — or a step_started can land in the log after step_completed has already
|
|
17
|
+
* written, producing unconsumed events on replay.
|
|
18
|
+
*
|
|
19
|
+
* Duplicate step_started events for a non-terminal step are still allowed
|
|
20
|
+
* (retries legitimately re-start a step), only writes to an already-terminal
|
|
21
|
+
* step are rejected.
|
|
22
|
+
*/
|
|
23
|
+
const stepLocks = new Map();
|
|
24
|
+
function withStepLock(key, fn) {
|
|
25
|
+
const prev = stepLocks.get(key);
|
|
26
|
+
const taskBox = {};
|
|
27
|
+
const task = (async () => {
|
|
28
|
+
if (prev) {
|
|
29
|
+
// Wait for the previous task to settle; don't inherit its errors.
|
|
30
|
+
await prev.catch(() => undefined);
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
return await fn();
|
|
34
|
+
}
|
|
35
|
+
finally {
|
|
36
|
+
if (stepLocks.get(key) === taskBox.task) {
|
|
37
|
+
stepLocks.delete(key);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
})();
|
|
41
|
+
taskBox.task = task;
|
|
42
|
+
stepLocks.set(key, task);
|
|
43
|
+
return task;
|
|
44
|
+
}
|
|
11
45
|
/**
|
|
12
46
|
* Helper function to delete all waits associated with a workflow run.
|
|
13
47
|
* Called when a run reaches a terminal state.
|
|
@@ -31,12 +65,11 @@ async function deleteAllWaitsForRun(basedir, runId) {
|
|
|
31
65
|
export function createEventsStorage(basedir, tag) {
|
|
32
66
|
return {
|
|
33
67
|
async create(runId, data, params) {
|
|
34
|
-
const eventId = `evnt_${monotonicUlid()}`;
|
|
35
|
-
const now = new Date();
|
|
36
68
|
// Validate request-supplied IDs before they're concatenated into
|
|
37
69
|
// filesystem paths. This is the primary defense against path traversal
|
|
38
70
|
// attacks where a client supplies runId / correlationId values like
|
|
39
71
|
// "../../../package" to read or write files outside the storage root.
|
|
72
|
+
// Run before taking the per-step mutex so malformed inputs fail fast.
|
|
40
73
|
if (runId != null && runId !== '') {
|
|
41
74
|
assertSafeEntityId('runId', runId);
|
|
42
75
|
}
|
|
@@ -45,653 +78,698 @@ export function createEventsStorage(basedir, tag) {
|
|
|
45
78
|
data.correlationId.length > 0) {
|
|
46
79
|
assertSafeEntityId('correlationId', data.correlationId);
|
|
47
80
|
}
|
|
48
|
-
//
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
81
|
+
// Step lifecycle events are serialized per-step via an in-process mutex
|
|
82
|
+
// so that the "check state, then write" sequence in step_started /
|
|
83
|
+
// step_completed / step_failed / step_retrying is atomic. step_created
|
|
84
|
+
// is also serialized so duplicate-create races don't leave extra
|
|
85
|
+
// step_created events in the log.
|
|
86
|
+
const isStepEvent = data.eventType === 'step_created' ||
|
|
87
|
+
data.eventType === 'step_started' ||
|
|
88
|
+
data.eventType === 'step_completed' ||
|
|
89
|
+
data.eventType === 'step_failed' ||
|
|
90
|
+
data.eventType === 'step_retrying';
|
|
91
|
+
if (isStepEvent && runId && data.correlationId) {
|
|
92
|
+
const lockKey = tag
|
|
93
|
+
? `${runId}-${data.correlationId}.${tag}`
|
|
94
|
+
: `${runId}-${data.correlationId}`;
|
|
95
|
+
return withStepLock(lockKey, () => createImpl());
|
|
52
96
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const validationError = validateUlidTimestamp(effectiveRunId, 'wrun_');
|
|
62
|
-
if (validationError) {
|
|
63
|
-
throw new WorkflowWorldError(validationError);
|
|
97
|
+
return createImpl();
|
|
98
|
+
async function createImpl() {
|
|
99
|
+
const eventId = `evnt_${monotonicUlid()}`;
|
|
100
|
+
const now = new Date();
|
|
101
|
+
// For run_created events, use client-provided runId or generate one server-side
|
|
102
|
+
let effectiveRunId;
|
|
103
|
+
if (data.eventType === 'run_created' && (!runId || runId === '')) {
|
|
104
|
+
effectiveRunId = `wrun_${monotonicUlid()}`;
|
|
64
105
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
//
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
updatedAt: now,
|
|
111
|
-
};
|
|
112
|
-
const runPath = taggedPath(basedir, 'runs', effectiveRunId, tag);
|
|
113
|
-
const created = await writeExclusive(runPath, JSON.stringify(createdRun, jsonReplacer));
|
|
114
|
-
if (created) {
|
|
115
|
-
// We created the run — also write the run_created event.
|
|
116
|
-
const runCreatedEventId = `evnt_${monotonicUlid()}`;
|
|
117
|
-
const runCreatedEvent = {
|
|
118
|
-
eventType: 'run_created',
|
|
106
|
+
else if (!runId) {
|
|
107
|
+
throw new Error('runId is required for non-run_created events');
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
effectiveRunId = runId;
|
|
111
|
+
}
|
|
112
|
+
// Validate client-provided runId timestamp is within acceptable threshold
|
|
113
|
+
if (data.eventType === 'run_created' && runId && runId !== '') {
|
|
114
|
+
const validationError = validateUlidTimestamp(effectiveRunId, 'wrun_');
|
|
115
|
+
if (validationError) {
|
|
116
|
+
throw new WorkflowWorldError(validationError);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
// specVersion is always sent by the runtime, but we provide a fallback for safety
|
|
120
|
+
const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
|
|
121
|
+
// Helper to check if run is in terminal state
|
|
122
|
+
const isRunTerminal = (status) => ['completed', 'failed', 'cancelled'].includes(status);
|
|
123
|
+
// Helper to check if step is in terminal state
|
|
124
|
+
const isStepTerminal = (status) => ['completed', 'failed', 'cancelled'].includes(status);
|
|
125
|
+
// Get current run state for validation (if not creating a new run)
|
|
126
|
+
// Skip run validation for step_completed and step_retrying - they only operate
|
|
127
|
+
// on running steps, and running steps are always allowed to modify regardless
|
|
128
|
+
// of run state. This optimization saves filesystem reads per step event.
|
|
129
|
+
let currentRun = null;
|
|
130
|
+
const skipRunValidationEvents = ['step_completed', 'step_retrying'];
|
|
131
|
+
if (data.eventType !== 'run_created' &&
|
|
132
|
+
!skipRunValidationEvents.includes(data.eventType)) {
|
|
133
|
+
currentRun = await readJSONWithFallback(basedir, 'runs', effectiveRunId, WorkflowRunSchema, tag);
|
|
134
|
+
// Resilient start: run_started on non-existent run with eventData
|
|
135
|
+
// creates the run first, so the queue can bootstrap a run that
|
|
136
|
+
// failed to create during start().
|
|
137
|
+
if (data.eventType === 'run_started' &&
|
|
138
|
+
!currentRun &&
|
|
139
|
+
'eventData' in data &&
|
|
140
|
+
data.eventData) {
|
|
141
|
+
const runInputData = data.eventData;
|
|
142
|
+
if (runInputData.deploymentId &&
|
|
143
|
+
runInputData.workflowName &&
|
|
144
|
+
runInputData.input !== undefined) {
|
|
145
|
+
// Atomically try to create the run entity. writeExclusive
|
|
146
|
+
// uses O_CREAT|O_EXCL so only the first writer wins,
|
|
147
|
+
// preventing a TOCTOU race where a concurrent run_created
|
|
148
|
+
// from start() could overwrite a run that was already
|
|
149
|
+
// transitioned to 'running'.
|
|
150
|
+
const createdRun = {
|
|
119
151
|
runId: effectiveRunId,
|
|
120
|
-
|
|
121
|
-
|
|
152
|
+
deploymentId: runInputData.deploymentId,
|
|
153
|
+
status: 'pending',
|
|
154
|
+
workflowName: runInputData.workflowName,
|
|
122
155
|
specVersion: effectiveSpecVersion,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
156
|
+
executionContext: runInputData.executionContext,
|
|
157
|
+
input: runInputData.input,
|
|
158
|
+
output: undefined,
|
|
159
|
+
error: undefined,
|
|
160
|
+
startedAt: undefined,
|
|
161
|
+
completedAt: undefined,
|
|
162
|
+
createdAt: now,
|
|
163
|
+
updatedAt: now,
|
|
129
164
|
};
|
|
130
|
-
const
|
|
131
|
-
await
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
165
|
+
const runPath = taggedPath(basedir, 'runs', effectiveRunId, tag);
|
|
166
|
+
const created = await writeExclusive(runPath, JSON.stringify(createdRun, jsonReplacer));
|
|
167
|
+
if (created) {
|
|
168
|
+
// We created the run — also write the run_created event.
|
|
169
|
+
const runCreatedEventId = `evnt_${monotonicUlid()}`;
|
|
170
|
+
const runCreatedEvent = {
|
|
171
|
+
eventType: 'run_created',
|
|
172
|
+
runId: effectiveRunId,
|
|
173
|
+
eventId: runCreatedEventId,
|
|
174
|
+
createdAt: now,
|
|
175
|
+
specVersion: effectiveSpecVersion,
|
|
176
|
+
eventData: {
|
|
177
|
+
deploymentId: runInputData.deploymentId,
|
|
178
|
+
workflowName: runInputData.workflowName,
|
|
179
|
+
input: runInputData.input,
|
|
180
|
+
executionContext: runInputData.executionContext,
|
|
181
|
+
},
|
|
182
|
+
};
|
|
183
|
+
const createdCompositeKey = `${effectiveRunId}-${runCreatedEventId}`;
|
|
184
|
+
await writeJSON(taggedPath(basedir, 'events', createdCompositeKey, tag), runCreatedEvent);
|
|
185
|
+
currentRun = createdRun;
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
// Run already exists (concurrent run_created won the
|
|
189
|
+
// race). Re-read it so downstream logic sees the real state.
|
|
190
|
+
currentRun = await readJSONWithFallback(basedir, 'runs', effectiveRunId, WorkflowRunSchema, tag);
|
|
191
|
+
}
|
|
138
192
|
}
|
|
139
193
|
}
|
|
140
194
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
if (currentRun) {
|
|
148
|
-
// Check if run requires a newer world version
|
|
149
|
-
if (requiresNewerWorld(currentRun.specVersion)) {
|
|
150
|
-
throw new RunNotSupportedError(currentRun.specVersion, SPEC_VERSION_CURRENT);
|
|
195
|
+
// run_failed on a non-existent run is rejected to match the
|
|
196
|
+
// postgres and vercel worlds, which both surface this as a
|
|
197
|
+
// WorkflowRunNotFoundError rather than silently persisting an
|
|
198
|
+
// event for a run that was never created.
|
|
199
|
+
if (data.eventType === 'run_failed' && !currentRun) {
|
|
200
|
+
throw new WorkflowRunNotFoundError(effectiveRunId);
|
|
151
201
|
}
|
|
152
|
-
//
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
];
|
|
167
|
-
// Idempotent operation: run_cancelled on already cancelled run is allowed
|
|
168
|
-
if (data.eventType === 'run_cancelled' &&
|
|
169
|
-
currentRun.status === 'cancelled') {
|
|
170
|
-
// Return existing state (idempotent)
|
|
171
|
-
const event = {
|
|
172
|
-
...data,
|
|
173
|
-
runId: effectiveRunId,
|
|
174
|
-
eventId,
|
|
175
|
-
createdAt: now,
|
|
176
|
-
specVersion: effectiveSpecVersion,
|
|
177
|
-
};
|
|
178
|
-
const compositeKey = `${effectiveRunId}-${eventId}`;
|
|
179
|
-
await writeJSON(taggedPath(basedir, 'events', compositeKey, tag), event);
|
|
180
|
-
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
181
|
-
return {
|
|
182
|
-
event: stripEventDataRefs(event, resolveData),
|
|
183
|
-
run: currentRun,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
// For run_started on terminal runs, use RunExpiredError so the
|
|
187
|
-
// runtime knows to exit without retrying.
|
|
188
|
-
if (data.eventType === 'run_started') {
|
|
189
|
-
throw new RunExpiredError(`Workflow run "${effectiveRunId}" is already in terminal state "${currentRun.status}"`);
|
|
190
|
-
}
|
|
191
|
-
// Other run state transitions are not allowed on terminal runs
|
|
192
|
-
if (runTerminalEvents.includes(data.eventType) ||
|
|
193
|
-
data.eventType === 'run_cancelled') {
|
|
194
|
-
throw new EntityConflictError(`Cannot transition run from terminal state "${currentRun.status}"`);
|
|
195
|
-
}
|
|
196
|
-
// Creating new entities on terminal runs is not allowed
|
|
197
|
-
if (data.eventType === 'step_created' ||
|
|
198
|
-
data.eventType === 'hook_created' ||
|
|
199
|
-
data.eventType === 'wait_created') {
|
|
200
|
-
throw new EntityConflictError(`Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
// Step-related event validation (ordering and terminal state)
|
|
204
|
-
// Store existingStep so we can reuse it later (avoid double read)
|
|
205
|
-
let validatedStep = null;
|
|
206
|
-
const stepEvents = [
|
|
207
|
-
'step_started',
|
|
208
|
-
'step_completed',
|
|
209
|
-
'step_failed',
|
|
210
|
-
'step_retrying',
|
|
211
|
-
];
|
|
212
|
-
if (stepEvents.includes(data.eventType) && data.correlationId) {
|
|
213
|
-
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
214
|
-
validatedStep = await readJSONWithFallback(basedir, 'steps', stepCompositeKey, StepSchema, tag);
|
|
215
|
-
// Event ordering: step must exist before these events
|
|
216
|
-
if (!validatedStep) {
|
|
217
|
-
throw new WorkflowWorldError(`Step "${data.correlationId}" not found`);
|
|
218
|
-
}
|
|
219
|
-
// Step terminal state validation
|
|
220
|
-
if (isStepTerminal(validatedStep.status)) {
|
|
221
|
-
throw new EntityConflictError(`Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
202
|
+
// ============================================================
|
|
203
|
+
// VERSION COMPATIBILITY: Check run spec version
|
|
204
|
+
// ============================================================
|
|
205
|
+
// For events that have fetched the run, check version compatibility.
|
|
206
|
+
// Skip for run_created (no existing run) and runtime events (step_completed, step_retrying).
|
|
207
|
+
if (currentRun) {
|
|
208
|
+
// Check if run requires a newer world version
|
|
209
|
+
if (requiresNewerWorld(currentRun.specVersion)) {
|
|
210
|
+
throw new RunNotSupportedError(currentRun.specVersion, SPEC_VERSION_CURRENT);
|
|
211
|
+
}
|
|
212
|
+
// Route to legacy handler for pre-event-sourcing runs
|
|
213
|
+
if (isLegacySpecVersion(currentRun.specVersion)) {
|
|
214
|
+
return handleLegacyEvent(basedir, effectiveRunId, data, currentRun, params);
|
|
215
|
+
}
|
|
222
216
|
}
|
|
223
|
-
//
|
|
217
|
+
// ============================================================
|
|
218
|
+
// VALIDATION: Terminal state and event ordering checks
|
|
219
|
+
// ============================================================
|
|
220
|
+
// Run terminal state validation
|
|
224
221
|
if (currentRun && isRunTerminal(currentRun.status)) {
|
|
225
|
-
|
|
226
|
-
|
|
222
|
+
const runTerminalEvents = [
|
|
223
|
+
'run_started',
|
|
224
|
+
'run_completed',
|
|
225
|
+
'run_failed',
|
|
226
|
+
];
|
|
227
|
+
// Idempotent operation: run_cancelled on already cancelled run is allowed
|
|
228
|
+
if (data.eventType === 'run_cancelled' &&
|
|
229
|
+
currentRun.status === 'cancelled') {
|
|
230
|
+
// Return existing state (idempotent)
|
|
231
|
+
const event = {
|
|
232
|
+
...data,
|
|
233
|
+
runId: effectiveRunId,
|
|
234
|
+
eventId,
|
|
235
|
+
createdAt: now,
|
|
236
|
+
specVersion: effectiveSpecVersion,
|
|
237
|
+
};
|
|
238
|
+
const compositeKey = `${effectiveRunId}-${eventId}`;
|
|
239
|
+
await writeJSON(taggedPath(basedir, 'events', compositeKey, tag), event);
|
|
240
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
241
|
+
return {
|
|
242
|
+
event: stripEventDataRefs(event, resolveData),
|
|
243
|
+
run: currentRun,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
// For run_started on terminal runs, use RunExpiredError so the
|
|
247
|
+
// runtime knows to exit without retrying.
|
|
248
|
+
if (data.eventType === 'run_started') {
|
|
249
|
+
throw new RunExpiredError(`Workflow run "${effectiveRunId}" is already in terminal state "${currentRun.status}"`);
|
|
250
|
+
}
|
|
251
|
+
// Other run state transitions are not allowed on terminal runs
|
|
252
|
+
if (runTerminalEvents.includes(data.eventType) ||
|
|
253
|
+
data.eventType === 'run_cancelled') {
|
|
254
|
+
throw new EntityConflictError(`Cannot transition run from terminal state "${currentRun.status}"`);
|
|
255
|
+
}
|
|
256
|
+
// Creating new entities on terminal runs is not allowed
|
|
257
|
+
if (data.eventType === 'step_created' ||
|
|
258
|
+
data.eventType === 'hook_created' ||
|
|
259
|
+
data.eventType === 'wait_created') {
|
|
260
|
+
throw new EntityConflictError(`Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
227
261
|
}
|
|
228
262
|
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
263
|
+
// Step-related event validation (ordering and terminal state)
|
|
264
|
+
// Store existingStep so we can reuse it later (avoid double read)
|
|
265
|
+
let validatedStep = null;
|
|
266
|
+
const stepEvents = [
|
|
267
|
+
'step_started',
|
|
268
|
+
'step_completed',
|
|
269
|
+
'step_failed',
|
|
270
|
+
'step_retrying',
|
|
271
|
+
];
|
|
272
|
+
if (stepEvents.includes(data.eventType) && data.correlationId) {
|
|
273
|
+
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
274
|
+
validatedStep = await readJSONWithFallback(basedir, 'steps', stepCompositeKey, StepSchema, tag);
|
|
275
|
+
// Event ordering: step must exist before these events
|
|
276
|
+
if (!validatedStep) {
|
|
277
|
+
throw new WorkflowWorldError(`Step "${data.correlationId}" not found`);
|
|
278
|
+
}
|
|
279
|
+
// Step terminal state validation
|
|
280
|
+
if (isStepTerminal(validatedStep.status)) {
|
|
281
|
+
throw new EntityConflictError(`Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
282
|
+
}
|
|
283
|
+
// On terminal runs: only allow completing/failing in-progress steps
|
|
284
|
+
if (currentRun && isRunTerminal(currentRun.status)) {
|
|
285
|
+
if (validatedStep.status !== 'running') {
|
|
286
|
+
throw new RunExpiredError(`Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
237
289
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
}
|
|
250
|
-
// Track entity created/updated for EventResult
|
|
251
|
-
let run;
|
|
252
|
-
let step;
|
|
253
|
-
let hook;
|
|
254
|
-
let wait;
|
|
255
|
-
// Create/update entity based on event type (event-sourced architecture)
|
|
256
|
-
// Run lifecycle events
|
|
257
|
-
if (data.eventType === 'run_created' && 'eventData' in data) {
|
|
258
|
-
const runData = data.eventData;
|
|
259
|
-
run = {
|
|
290
|
+
// Hook-related event validation (ordering)
|
|
291
|
+
const hookEventsRequiringExistence = ['hook_disposed', 'hook_received'];
|
|
292
|
+
if (hookEventsRequiringExistence.includes(data.eventType) &&
|
|
293
|
+
data.correlationId) {
|
|
294
|
+
const existingHook = await readJSONWithFallback(basedir, 'hooks', data.correlationId, HookSchema, tag);
|
|
295
|
+
if (!existingHook) {
|
|
296
|
+
throw new HookNotFoundError(data.correlationId);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const event = {
|
|
300
|
+
...data,
|
|
260
301
|
runId: effectiveRunId,
|
|
261
|
-
|
|
262
|
-
status: 'pending',
|
|
263
|
-
workflowName: runData.workflowName,
|
|
264
|
-
// Propagate specVersion from the event to the run entity
|
|
265
|
-
specVersion: effectiveSpecVersion,
|
|
266
|
-
executionContext: runData.executionContext,
|
|
267
|
-
input: runData.input,
|
|
268
|
-
output: undefined,
|
|
269
|
-
error: undefined,
|
|
270
|
-
startedAt: undefined,
|
|
271
|
-
completedAt: undefined,
|
|
302
|
+
eventId,
|
|
272
303
|
createdAt: now,
|
|
273
|
-
|
|
304
|
+
specVersion: effectiveSpecVersion,
|
|
274
305
|
};
|
|
275
|
-
//
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
// in duplicate run_created events in the event log.
|
|
279
|
-
const runPath = taggedPath(basedir, 'runs', effectiveRunId, tag);
|
|
280
|
-
const created = await writeExclusive(runPath, JSON.stringify(run, jsonReplacer, 2));
|
|
281
|
-
if (!created) {
|
|
282
|
-
throw new EntityConflictError(`Workflow run "${effectiveRunId}" already exists`);
|
|
306
|
+
// Strip eventData from run_started — it belongs on run_created only.
|
|
307
|
+
if (data.eventType === 'run_started' && 'eventData' in event) {
|
|
308
|
+
delete event.eventData;
|
|
283
309
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
if (currentRun.status === 'running') {
|
|
294
|
-
return { run: currentRun };
|
|
295
|
-
}
|
|
310
|
+
// Track entity created/updated for EventResult
|
|
311
|
+
let run;
|
|
312
|
+
let step;
|
|
313
|
+
let hook;
|
|
314
|
+
let wait;
|
|
315
|
+
// Create/update entity based on event type (event-sourced architecture)
|
|
316
|
+
// Run lifecycle events
|
|
317
|
+
if (data.eventType === 'run_created' && 'eventData' in data) {
|
|
318
|
+
const runData = data.eventData;
|
|
296
319
|
run = {
|
|
297
|
-
runId:
|
|
298
|
-
deploymentId:
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
status: 'running',
|
|
320
|
+
runId: effectiveRunId,
|
|
321
|
+
deploymentId: runData.deploymentId,
|
|
322
|
+
status: 'pending',
|
|
323
|
+
workflowName: runData.workflowName,
|
|
324
|
+
// Propagate specVersion from the event to the run entity
|
|
325
|
+
specVersion: effectiveSpecVersion,
|
|
326
|
+
executionContext: runData.executionContext,
|
|
327
|
+
input: runData.input,
|
|
306
328
|
output: undefined,
|
|
307
329
|
error: undefined,
|
|
330
|
+
startedAt: undefined,
|
|
308
331
|
completedAt: undefined,
|
|
309
|
-
|
|
332
|
+
createdAt: now,
|
|
310
333
|
updatedAt: now,
|
|
311
334
|
};
|
|
312
|
-
|
|
335
|
+
// Use writeExclusive (O_CREAT|O_EXCL) to atomically create the
|
|
336
|
+
// run entity file. This prevents a TOCTOU race with the resilient
|
|
337
|
+
// start path (run_started on non-existent run) that could result
|
|
338
|
+
// in duplicate run_created events in the event log.
|
|
339
|
+
const runPath = taggedPath(basedir, 'runs', effectiveRunId, tag);
|
|
340
|
+
const created = await writeExclusive(runPath, JSON.stringify(run, jsonReplacer, 2));
|
|
341
|
+
if (!created) {
|
|
342
|
+
throw new EntityConflictError(`Workflow run "${effectiveRunId}" already exists`);
|
|
343
|
+
}
|
|
313
344
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
345
|
+
else if (data.eventType === 'run_started') {
|
|
346
|
+
// Reuse currentRun from validation (already read above)
|
|
347
|
+
if (currentRun) {
|
|
348
|
+
// If already running, return the run without inserting a
|
|
349
|
+
// duplicate event. This makes run_started idempotent for
|
|
350
|
+
// concurrent invocations. We omit preloaded events here
|
|
351
|
+
// because this is a rare race-condition path — the runtime
|
|
352
|
+
// falls back to loadWorkflowRunEvents().
|
|
353
|
+
if (currentRun.status === 'running') {
|
|
354
|
+
return { run: currentRun };
|
|
355
|
+
}
|
|
356
|
+
run = {
|
|
357
|
+
runId: currentRun.runId,
|
|
358
|
+
deploymentId: currentRun.deploymentId,
|
|
359
|
+
workflowName: currentRun.workflowName,
|
|
360
|
+
specVersion: currentRun.specVersion,
|
|
361
|
+
executionContext: currentRun.executionContext,
|
|
362
|
+
input: currentRun.input,
|
|
363
|
+
createdAt: currentRun.createdAt,
|
|
364
|
+
expiredAt: currentRun.expiredAt,
|
|
365
|
+
status: 'running',
|
|
366
|
+
output: undefined,
|
|
367
|
+
error: undefined,
|
|
368
|
+
completedAt: undefined,
|
|
369
|
+
startedAt: currentRun.startedAt ?? now,
|
|
370
|
+
updatedAt: now,
|
|
371
|
+
};
|
|
372
|
+
await writeJSON(taggedPath(basedir, 'runs', effectiveRunId, tag), run, { overwrite: true });
|
|
373
|
+
}
|
|
340
374
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
};
|
|
368
|
-
await writeJSON(taggedPath(basedir, 'runs', effectiveRunId, tag), run, { overwrite: true });
|
|
369
|
-
await Promise.all([
|
|
370
|
-
deleteAllHooksForRun(basedir, effectiveRunId),
|
|
371
|
-
deleteAllWaitsForRun(basedir, effectiveRunId),
|
|
372
|
-
]);
|
|
375
|
+
else if (data.eventType === 'run_completed' && 'eventData' in data) {
|
|
376
|
+
const completedData = data.eventData;
|
|
377
|
+
// Reuse currentRun from validation (already read above)
|
|
378
|
+
if (currentRun) {
|
|
379
|
+
run = {
|
|
380
|
+
runId: currentRun.runId,
|
|
381
|
+
deploymentId: currentRun.deploymentId,
|
|
382
|
+
workflowName: currentRun.workflowName,
|
|
383
|
+
specVersion: currentRun.specVersion,
|
|
384
|
+
executionContext: currentRun.executionContext,
|
|
385
|
+
input: currentRun.input,
|
|
386
|
+
createdAt: currentRun.createdAt,
|
|
387
|
+
expiredAt: currentRun.expiredAt,
|
|
388
|
+
startedAt: currentRun.startedAt,
|
|
389
|
+
status: 'completed',
|
|
390
|
+
output: completedData.output,
|
|
391
|
+
error: undefined,
|
|
392
|
+
completedAt: now,
|
|
393
|
+
updatedAt: now,
|
|
394
|
+
};
|
|
395
|
+
await writeJSON(taggedPath(basedir, 'runs', effectiveRunId, tag), run, { overwrite: true });
|
|
396
|
+
await Promise.all([
|
|
397
|
+
deleteAllHooksForRun(basedir, effectiveRunId),
|
|
398
|
+
deleteAllWaitsForRun(basedir, effectiveRunId),
|
|
399
|
+
]);
|
|
400
|
+
}
|
|
373
401
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
402
|
+
else if (data.eventType === 'run_failed' && 'eventData' in data) {
|
|
403
|
+
const failedData = data.eventData;
|
|
404
|
+
// Reuse currentRun from validation (already read above)
|
|
405
|
+
if (currentRun) {
|
|
406
|
+
// The error field is SerializedData (Uint8Array) produced by
|
|
407
|
+
// dehydrateRunError. We store it verbatim — consumers hydrate it
|
|
408
|
+
// via hydrateRunError to reconstruct the original thrown value.
|
|
409
|
+
run = {
|
|
410
|
+
runId: currentRun.runId,
|
|
411
|
+
deploymentId: currentRun.deploymentId,
|
|
412
|
+
workflowName: currentRun.workflowName,
|
|
413
|
+
specVersion: currentRun.specVersion,
|
|
414
|
+
executionContext: currentRun.executionContext,
|
|
415
|
+
input: currentRun.input,
|
|
416
|
+
createdAt: currentRun.createdAt,
|
|
417
|
+
expiredAt: currentRun.expiredAt,
|
|
418
|
+
startedAt: currentRun.startedAt,
|
|
419
|
+
status: 'failed',
|
|
420
|
+
output: undefined,
|
|
421
|
+
error: failedData.error,
|
|
422
|
+
errorCode: failedData.errorCode,
|
|
423
|
+
completedAt: now,
|
|
424
|
+
updatedAt: now,
|
|
425
|
+
};
|
|
426
|
+
await writeJSON(taggedPath(basedir, 'runs', effectiveRunId, tag), run, { overwrite: true });
|
|
427
|
+
await Promise.all([
|
|
428
|
+
deleteAllHooksForRun(basedir, effectiveRunId),
|
|
429
|
+
deleteAllWaitsForRun(basedir, effectiveRunId),
|
|
430
|
+
]);
|
|
431
|
+
}
|
|
399
432
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step);
|
|
425
|
-
}
|
|
426
|
-
else if (data.eventType === 'step_started') {
|
|
427
|
-
// step_started: Increments attempt, sets status to 'running'
|
|
428
|
-
// Sets startedAt only on the first start (not updated on retries)
|
|
429
|
-
// Reuse validatedStep from validation (already read above)
|
|
430
|
-
if (validatedStep) {
|
|
431
|
-
// Check if retryAfter timestamp hasn't been reached yet
|
|
432
|
-
if (validatedStep.retryAfter &&
|
|
433
|
-
validatedStep.retryAfter.getTime() > Date.now()) {
|
|
434
|
-
throw new TooEarlyError(`Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, {
|
|
435
|
-
retryAfter: Math.ceil((validatedStep.retryAfter.getTime() - Date.now()) / 1000),
|
|
436
|
-
});
|
|
433
|
+
else if (data.eventType === 'run_cancelled') {
|
|
434
|
+
// Reuse currentRun from validation (already read above)
|
|
435
|
+
if (currentRun) {
|
|
436
|
+
run = {
|
|
437
|
+
runId: currentRun.runId,
|
|
438
|
+
deploymentId: currentRun.deploymentId,
|
|
439
|
+
workflowName: currentRun.workflowName,
|
|
440
|
+
specVersion: currentRun.specVersion,
|
|
441
|
+
executionContext: currentRun.executionContext,
|
|
442
|
+
input: currentRun.input,
|
|
443
|
+
createdAt: currentRun.createdAt,
|
|
444
|
+
expiredAt: currentRun.expiredAt,
|
|
445
|
+
startedAt: currentRun.startedAt,
|
|
446
|
+
status: 'cancelled',
|
|
447
|
+
output: undefined,
|
|
448
|
+
error: undefined,
|
|
449
|
+
completedAt: now,
|
|
450
|
+
updatedAt: now,
|
|
451
|
+
};
|
|
452
|
+
await writeJSON(taggedPath(basedir, 'runs', effectiveRunId, tag), run, { overwrite: true });
|
|
453
|
+
await Promise.all([
|
|
454
|
+
deleteAllHooksForRun(basedir, effectiveRunId),
|
|
455
|
+
deleteAllWaitsForRun(basedir, effectiveRunId),
|
|
456
|
+
]);
|
|
437
457
|
}
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
458
|
+
}
|
|
459
|
+
else if (
|
|
460
|
+
// Step lifecycle events
|
|
461
|
+
data.eventType === 'step_created' &&
|
|
462
|
+
'eventData' in data) {
|
|
463
|
+
// step_created: Creates step entity with status 'pending', attempt=0, createdAt set.
|
|
464
|
+
// Two concurrent invocations with identical correlationIds (e.g. the
|
|
465
|
+
// snapshot runtime's deterministic correlationIds across replays)
|
|
466
|
+
// must be deduped — otherwise both writes succeed and the event log
|
|
467
|
+
// ends up with duplicate step_created entries. The outer
|
|
468
|
+
// withStepLock mutex serializes within a single process; this
|
|
469
|
+
// O_CREAT|O_EXCL constraint file additionally protects against
|
|
470
|
+
// cross-process races (two pnpm workers, redelivered queue
|
|
471
|
+
// messages, etc.). The loser throws EntityConflictError so the
|
|
472
|
+
// runtime's existing catch path can swallow it and avoid
|
|
473
|
+
// double-queuing the step.
|
|
474
|
+
const stepCreatedLockName = tag
|
|
475
|
+
? `${effectiveRunId}-${data.correlationId}.created.${tag}`
|
|
476
|
+
: `${effectiveRunId}-${data.correlationId}.created`;
|
|
477
|
+
const stepCreatedLockPath = resolveWithinBase(basedir, '.locks', 'steps', stepCreatedLockName);
|
|
478
|
+
const stepCreatedClaimed = await writeExclusive(stepCreatedLockPath, '');
|
|
479
|
+
if (!stepCreatedClaimed) {
|
|
480
|
+
throw new EntityConflictError(`Step "${data.correlationId}" already created`);
|
|
447
481
|
}
|
|
482
|
+
const stepData = data.eventData;
|
|
448
483
|
step = {
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
484
|
+
runId: effectiveRunId,
|
|
485
|
+
stepId: data.correlationId,
|
|
486
|
+
stepName: stepData.stepName,
|
|
487
|
+
status: 'pending',
|
|
488
|
+
input: stepData.input,
|
|
489
|
+
output: undefined,
|
|
490
|
+
error: undefined,
|
|
491
|
+
attempt: 0,
|
|
492
|
+
startedAt: undefined,
|
|
493
|
+
completedAt: undefined,
|
|
494
|
+
createdAt: now,
|
|
457
495
|
updatedAt: now,
|
|
496
|
+
// Propagate specVersion from the event to the step entity
|
|
497
|
+
specVersion: effectiveSpecVersion,
|
|
458
498
|
};
|
|
459
|
-
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
else if (data.eventType === 'step_completed' && 'eventData' in data) {
|
|
463
|
-
// step_completed: Terminal state with output
|
|
464
|
-
// Uses writeExclusive on a lock file to atomically prevent concurrent
|
|
465
|
-
// invocations from both completing the same step (TOCTOU race).
|
|
466
|
-
const completedData = data.eventData;
|
|
467
|
-
if (validatedStep) {
|
|
468
499
|
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
500
|
+
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step);
|
|
501
|
+
}
|
|
502
|
+
else if (data.eventType === 'step_started') {
|
|
503
|
+
// step_started: Increments attempt, sets status to 'running'
|
|
504
|
+
// Sets startedAt only on the first start (not updated on retries)
|
|
505
|
+
// Reuse validatedStep from validation (already read above)
|
|
506
|
+
if (validatedStep) {
|
|
507
|
+
// Check if retryAfter timestamp hasn't been reached yet
|
|
508
|
+
if (validatedStep.retryAfter &&
|
|
509
|
+
validatedStep.retryAfter.getTime() > Date.now()) {
|
|
510
|
+
throw new TooEarlyError(`Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, {
|
|
511
|
+
retryAfter: Math.ceil((validatedStep.retryAfter.getTime() - Date.now()) / 1000),
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
// Best-effort guard: re-read the step entity to check if it
|
|
515
|
+
// reached terminal state between the validation read and now.
|
|
516
|
+
// This narrows the TOCTOU window but does not fully eliminate it
|
|
517
|
+
// (the local world is single-process / dev-only; the postgres
|
|
518
|
+
// world uses SQL-level atomic guards for production).
|
|
519
|
+
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
520
|
+
const freshStep = await readJSONWithFallback(basedir, 'steps', stepCompositeKey, StepSchema, tag);
|
|
521
|
+
if (freshStep && isStepTerminal(freshStep.status)) {
|
|
522
|
+
throw new EntityConflictError(`Cannot modify step in terminal state "${freshStep.status}"`);
|
|
523
|
+
}
|
|
524
|
+
step = {
|
|
525
|
+
...validatedStep,
|
|
526
|
+
status: 'running',
|
|
527
|
+
// Only set startedAt on the first start
|
|
528
|
+
startedAt: validatedStep.startedAt ?? now,
|
|
529
|
+
// Increment attempt counter on every start
|
|
530
|
+
attempt: validatedStep.attempt + 1,
|
|
531
|
+
// Clear retryAfter now that the step has started
|
|
532
|
+
retryAfter: undefined,
|
|
533
|
+
updatedAt: now,
|
|
534
|
+
};
|
|
535
|
+
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
476
536
|
}
|
|
477
|
-
step = {
|
|
478
|
-
...validatedStep,
|
|
479
|
-
status: 'completed',
|
|
480
|
-
output: completedData.result,
|
|
481
|
-
completedAt: now,
|
|
482
|
-
updatedAt: now,
|
|
483
|
-
};
|
|
484
|
-
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
485
537
|
}
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
538
|
+
else if (data.eventType === 'step_completed' && 'eventData' in data) {
|
|
539
|
+
// step_completed: Terminal state with output
|
|
540
|
+
// Uses writeExclusive on a lock file to atomically prevent concurrent
|
|
541
|
+
// invocations from both completing the same step (TOCTOU race).
|
|
542
|
+
const completedData = data.eventData;
|
|
543
|
+
if (validatedStep) {
|
|
544
|
+
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
545
|
+
const lockName = tag
|
|
546
|
+
? `${stepCompositeKey}.terminal.${tag}`
|
|
547
|
+
: `${stepCompositeKey}.terminal`;
|
|
548
|
+
const terminalLockPath = resolveWithinBase(basedir, '.locks', 'steps', lockName);
|
|
549
|
+
const claimed = await writeExclusive(terminalLockPath, '');
|
|
550
|
+
if (!claimed) {
|
|
551
|
+
throw new EntityConflictError('Cannot modify step in terminal state');
|
|
552
|
+
}
|
|
553
|
+
step = {
|
|
554
|
+
...validatedStep,
|
|
555
|
+
status: 'completed',
|
|
556
|
+
output: completedData.result,
|
|
557
|
+
completedAt: now,
|
|
558
|
+
updatedAt: now,
|
|
559
|
+
};
|
|
560
|
+
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
501
561
|
}
|
|
502
|
-
const error = {
|
|
503
|
-
message: typeof failedData.error === 'string'
|
|
504
|
-
? failedData.error
|
|
505
|
-
: (failedData.error?.message ?? 'Unknown error'),
|
|
506
|
-
stack: failedData.stack,
|
|
507
|
-
};
|
|
508
|
-
step = {
|
|
509
|
-
...validatedStep,
|
|
510
|
-
status: 'failed',
|
|
511
|
-
error,
|
|
512
|
-
completedAt: now,
|
|
513
|
-
updatedAt: now,
|
|
514
|
-
};
|
|
515
|
-
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
516
562
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
563
|
+
else if (data.eventType === 'step_failed' && 'eventData' in data) {
|
|
564
|
+
// step_failed: Terminal state with error
|
|
565
|
+
// Uses writeExclusive on a lock file to atomically prevent concurrent
|
|
566
|
+
// invocations from both failing the same step (TOCTOU race).
|
|
567
|
+
const failedData = data.eventData;
|
|
568
|
+
if (validatedStep) {
|
|
569
|
+
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
570
|
+
const lockName = tag
|
|
571
|
+
? `${stepCompositeKey}.terminal.${tag}`
|
|
572
|
+
: `${stepCompositeKey}.terminal`;
|
|
573
|
+
const terminalLockPath = resolveWithinBase(basedir, '.locks', 'steps', lockName);
|
|
574
|
+
const claimed = await writeExclusive(terminalLockPath, '');
|
|
575
|
+
if (!claimed) {
|
|
576
|
+
throw new EntityConflictError('Cannot modify step in terminal state');
|
|
577
|
+
}
|
|
578
|
+
// The error field is SerializedData (Uint8Array) produced by
|
|
579
|
+
// dehydrateStepError. We store it verbatim — consumers hydrate it
|
|
580
|
+
// via hydrateStepError to reconstruct the original thrown value.
|
|
581
|
+
step = {
|
|
582
|
+
...validatedStep,
|
|
583
|
+
status: 'failed',
|
|
584
|
+
error: failedData.error,
|
|
585
|
+
completedAt: now,
|
|
586
|
+
updatedAt: now,
|
|
587
|
+
};
|
|
588
|
+
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
589
|
+
}
|
|
537
590
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
591
|
+
else if (data.eventType === 'step_retrying' && 'eventData' in data) {
|
|
592
|
+
// step_retrying: Sets status back to 'pending', records error
|
|
593
|
+
// Reuse validatedStep from validation (already read above)
|
|
594
|
+
const retryData = data.eventData;
|
|
595
|
+
if (validatedStep) {
|
|
596
|
+
const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
597
|
+
step = {
|
|
598
|
+
...validatedStep,
|
|
599
|
+
status: 'pending',
|
|
600
|
+
error: retryData.error,
|
|
601
|
+
retryAfter: retryData.retryAfter,
|
|
602
|
+
updatedAt: now,
|
|
603
|
+
};
|
|
604
|
+
await writeJSON(taggedPath(basedir, 'steps', stepCompositeKey, tag), step, { overwrite: true });
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
else if (
|
|
608
|
+
// Hook lifecycle events
|
|
609
|
+
data.eventType === 'hook_created' &&
|
|
610
|
+
'eventData' in data) {
|
|
611
|
+
const hookData = data.eventData;
|
|
612
|
+
// Atomically claim the token using an exclusive-create constraint file.
|
|
613
|
+
// This avoids the TOCTOU race of the previous read-all-then-check approach.
|
|
614
|
+
const constraintPath = path.join(basedir, 'hooks', 'tokens', `${hashToken(hookData.token)}.json`);
|
|
615
|
+
const tokenClaimed = await writeExclusive(constraintPath, JSON.stringify({
|
|
616
|
+
token: hookData.token,
|
|
617
|
+
hookId: data.correlationId,
|
|
561
618
|
runId: effectiveRunId,
|
|
562
|
-
|
|
619
|
+
}));
|
|
620
|
+
if (!tokenClaimed) {
|
|
621
|
+
// Create hook_conflict event instead of hook_created
|
|
622
|
+
// This allows the workflow to continue and fail gracefully when the hook is awaited
|
|
623
|
+
const conflictEvent = {
|
|
624
|
+
eventType: 'hook_conflict',
|
|
625
|
+
correlationId: data.correlationId,
|
|
626
|
+
eventData: {
|
|
627
|
+
token: hookData.token,
|
|
628
|
+
},
|
|
629
|
+
runId: effectiveRunId,
|
|
630
|
+
eventId,
|
|
631
|
+
createdAt: now,
|
|
632
|
+
specVersion: effectiveSpecVersion,
|
|
633
|
+
};
|
|
634
|
+
// Store the conflict event
|
|
635
|
+
const compositeKey = `${effectiveRunId}-${eventId}`;
|
|
636
|
+
await writeJSON(taggedPath(basedir, 'events', compositeKey, tag), conflictEvent);
|
|
637
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
638
|
+
const filteredEvent = stripEventDataRefs(conflictEvent, resolveData);
|
|
639
|
+
// Return EventResult with conflict event (no hook entity created)
|
|
640
|
+
return {
|
|
641
|
+
event: filteredEvent,
|
|
642
|
+
run,
|
|
643
|
+
step,
|
|
644
|
+
hook: undefined,
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
hook = {
|
|
648
|
+
runId: effectiveRunId,
|
|
649
|
+
hookId: data.correlationId,
|
|
650
|
+
token: hookData.token,
|
|
651
|
+
metadata: hookData.metadata,
|
|
652
|
+
ownerId: 'local-owner',
|
|
653
|
+
projectId: 'local-project',
|
|
654
|
+
environment: 'local',
|
|
563
655
|
createdAt: now,
|
|
656
|
+
// Propagate specVersion from the event to the hook entity
|
|
564
657
|
specVersion: effectiveSpecVersion,
|
|
658
|
+
isWebhook: hookData.isWebhook ?? false,
|
|
659
|
+
isSystem: hookData.isSystem ?? false,
|
|
565
660
|
};
|
|
566
|
-
|
|
567
|
-
const compositeKey = `${effectiveRunId}-${eventId}`;
|
|
568
|
-
await writeJSON(taggedPath(basedir, 'events', compositeKey, tag), conflictEvent);
|
|
569
|
-
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
570
|
-
const filteredEvent = stripEventDataRefs(conflictEvent, resolveData);
|
|
571
|
-
// Return EventResult with conflict event (no hook entity created)
|
|
572
|
-
return {
|
|
573
|
-
event: filteredEvent,
|
|
574
|
-
run,
|
|
575
|
-
step,
|
|
576
|
-
hook: undefined,
|
|
577
|
-
};
|
|
661
|
+
await writeJSON(taggedPath(basedir, 'hooks', data.correlationId, tag), hook);
|
|
578
662
|
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
: `${data.correlationId}.disposed`;
|
|
601
|
-
const lockPath = resolveWithinBase(basedir, '.locks', 'hooks', hookLockName);
|
|
602
|
-
const claimed = await writeExclusive(lockPath, '');
|
|
603
|
-
if (!claimed) {
|
|
604
|
-
throw new EntityConflictError(`Hook "${data.correlationId}" already disposed`);
|
|
605
|
-
}
|
|
606
|
-
// Read the hook to get its token before deleting
|
|
607
|
-
const hookPath = taggedPath(basedir, 'hooks', data.correlationId, tag);
|
|
608
|
-
const existingHook = await readJSONWithFallback(basedir, 'hooks', data.correlationId, HookSchema, tag);
|
|
609
|
-
if (existingHook) {
|
|
610
|
-
// Delete the token constraint file to free up the token for reuse
|
|
611
|
-
const disposedConstraintPath = path.join(basedir, 'hooks', 'tokens', `${hashToken(existingHook.token)}.json`);
|
|
612
|
-
await deleteJSON(disposedConstraintPath);
|
|
663
|
+
else if (data.eventType === 'hook_disposed') {
|
|
664
|
+
// hook_disposed: Deletes hook entity, rejects duplicates.
|
|
665
|
+
// Uses writeExclusive on a lock file to atomically prevent concurrent
|
|
666
|
+
// invocations from both disposing the same hook (TOCTOU race).
|
|
667
|
+
const hookLockName = tag
|
|
668
|
+
? `${data.correlationId}.disposed.${tag}`
|
|
669
|
+
: `${data.correlationId}.disposed`;
|
|
670
|
+
const lockPath = resolveWithinBase(basedir, '.locks', 'hooks', hookLockName);
|
|
671
|
+
const claimed = await writeExclusive(lockPath, '');
|
|
672
|
+
if (!claimed) {
|
|
673
|
+
throw new EntityConflictError(`Hook "${data.correlationId}" already disposed`);
|
|
674
|
+
}
|
|
675
|
+
// Read the hook to get its token before deleting
|
|
676
|
+
const hookPath = taggedPath(basedir, 'hooks', data.correlationId, tag);
|
|
677
|
+
const existingHook = await readJSONWithFallback(basedir, 'hooks', data.correlationId, HookSchema, tag);
|
|
678
|
+
if (existingHook) {
|
|
679
|
+
// Delete the token constraint file to free up the token for reuse
|
|
680
|
+
const disposedConstraintPath = path.join(basedir, 'hooks', 'tokens', `${hashToken(existingHook.token)}.json`);
|
|
681
|
+
await deleteJSON(disposedConstraintPath);
|
|
682
|
+
}
|
|
683
|
+
await deleteJSON(hookPath);
|
|
613
684
|
}
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
685
|
+
else if (data.eventType === 'wait_created' && 'eventData' in data) {
|
|
686
|
+
// wait_created: Creates wait entity with status 'waiting'.
|
|
687
|
+
// Atomic claim on a per-(runId, correlationId) constraint file
|
|
688
|
+
// ensures duplicate wait_created from concurrent invocations
|
|
689
|
+
// surfaces as EntityConflictError (replaces a prior TOCTOU
|
|
690
|
+
// read-then-check that could let both writers through).
|
|
691
|
+
const waitCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
692
|
+
const waitCreatedLockName = tag
|
|
693
|
+
? `${waitCompositeKey}.created.${tag}`
|
|
694
|
+
: `${waitCompositeKey}.created`;
|
|
695
|
+
const waitCreatedLockPath = resolveWithinBase(basedir, '.locks', 'waits', waitCreatedLockName);
|
|
696
|
+
const waitCreatedClaimed = await writeExclusive(waitCreatedLockPath, '');
|
|
697
|
+
if (!waitCreatedClaimed) {
|
|
698
|
+
throw new EntityConflictError(`Wait "${data.correlationId}" already exists`);
|
|
699
|
+
}
|
|
700
|
+
const waitData = data.eventData;
|
|
701
|
+
wait = {
|
|
702
|
+
waitId: waitCompositeKey,
|
|
703
|
+
runId: effectiveRunId,
|
|
704
|
+
status: 'waiting',
|
|
705
|
+
resumeAt: waitData.resumeAt,
|
|
706
|
+
completedAt: undefined,
|
|
707
|
+
createdAt: now,
|
|
708
|
+
updatedAt: now,
|
|
709
|
+
specVersion: effectiveSpecVersion,
|
|
710
|
+
};
|
|
711
|
+
await writeJSON(taggedPath(basedir, 'waits', waitCompositeKey, tag), wait);
|
|
623
712
|
}
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
713
|
+
else if (data.eventType === 'wait_completed') {
|
|
714
|
+
// wait_completed: Transitions wait to 'completed', rejects duplicates.
|
|
715
|
+
// Uses writeExclusive on a lock file to atomically prevent concurrent
|
|
716
|
+
// invocations from both completing the same wait (TOCTOU race).
|
|
717
|
+
const waitCompositeKey = `${effectiveRunId}-${data.correlationId}`;
|
|
718
|
+
const waitLockName = tag
|
|
719
|
+
? `${waitCompositeKey}.completed.${tag}`
|
|
720
|
+
: `${waitCompositeKey}.completed`;
|
|
721
|
+
const lockPath = resolveWithinBase(basedir, '.locks', 'waits', waitLockName);
|
|
722
|
+
const claimed = await writeExclusive(lockPath, '');
|
|
723
|
+
if (!claimed) {
|
|
724
|
+
throw new EntityConflictError(`Wait "${data.correlationId}" already completed`);
|
|
725
|
+
}
|
|
726
|
+
const existingWait = await readJSONWithFallback(basedir, 'waits', waitCompositeKey, WaitSchema, tag);
|
|
727
|
+
if (!existingWait) {
|
|
728
|
+
// Clean up the lock file we just claimed — the wait doesn't exist
|
|
729
|
+
await fs.unlink(lockPath).catch(() => { });
|
|
730
|
+
throw new WorkflowWorldError(`Wait "${data.correlationId}" not found`);
|
|
731
|
+
}
|
|
732
|
+
// The lock file (writeExclusive above) already prevents concurrent
|
|
733
|
+
// completions — no additional status check needed.
|
|
734
|
+
wait = {
|
|
735
|
+
...existingWait,
|
|
736
|
+
status: 'completed',
|
|
737
|
+
completedAt: now,
|
|
738
|
+
updatedAt: now,
|
|
739
|
+
};
|
|
740
|
+
await writeJSON(taggedPath(basedir, 'waits', waitCompositeKey, tag), wait, { overwrite: true });
|
|
648
741
|
}
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
742
|
+
// Note: hook_received events are stored in the event log but don't
|
|
743
|
+
// modify the Hook entity (which doesn't have a payload field)
|
|
744
|
+
// Store event using composite key {runId}-{eventId}
|
|
745
|
+
const compositeKey = `${effectiveRunId}-${eventId}`;
|
|
746
|
+
await writeJSON(taggedPath(basedir, 'events', compositeKey, tag), event);
|
|
747
|
+
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
748
|
+
const filteredEvent = stripEventDataRefs(event, resolveData);
|
|
749
|
+
// For run_started: include all events so the runtime can skip
|
|
750
|
+
// the initial events.list call and reduce TTFB.
|
|
751
|
+
let events;
|
|
752
|
+
if (data.eventType === 'run_started' && run) {
|
|
753
|
+
const allEvents = await paginatedFileSystemQuery({
|
|
754
|
+
directory: path.join(basedir, 'events'),
|
|
755
|
+
schema: EventSchema,
|
|
756
|
+
filePrefix: `${effectiveRunId}-`,
|
|
757
|
+
sortOrder: 'asc',
|
|
758
|
+
getCreatedAt: getObjectCreatedAt('evnt'),
|
|
759
|
+
getId: (e) => e.eventId,
|
|
760
|
+
});
|
|
761
|
+
events = allEvents.data;
|
|
654
762
|
}
|
|
655
|
-
//
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
763
|
+
// Return EventResult with event and any created/updated entity
|
|
764
|
+
return {
|
|
765
|
+
event: filteredEvent,
|
|
766
|
+
run,
|
|
767
|
+
step,
|
|
768
|
+
hook,
|
|
769
|
+
wait,
|
|
770
|
+
events,
|
|
662
771
|
};
|
|
663
|
-
|
|
664
|
-
}
|
|
665
|
-
// Note: hook_received events are stored in the event log but don't
|
|
666
|
-
// modify the Hook entity (which doesn't have a payload field)
|
|
667
|
-
// Store event using composite key {runId}-{eventId}
|
|
668
|
-
const compositeKey = `${effectiveRunId}-${eventId}`;
|
|
669
|
-
await writeJSON(taggedPath(basedir, 'events', compositeKey, tag), event);
|
|
670
|
-
const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION;
|
|
671
|
-
const filteredEvent = stripEventDataRefs(event, resolveData);
|
|
672
|
-
// For run_started: include all events so the runtime can skip
|
|
673
|
-
// the initial events.list call and reduce TTFB.
|
|
674
|
-
let events;
|
|
675
|
-
if (data.eventType === 'run_started' && run) {
|
|
676
|
-
const allEvents = await paginatedFileSystemQuery({
|
|
677
|
-
directory: path.join(basedir, 'events'),
|
|
678
|
-
schema: EventSchema,
|
|
679
|
-
filePrefix: `${effectiveRunId}-`,
|
|
680
|
-
sortOrder: 'asc',
|
|
681
|
-
getCreatedAt: getObjectCreatedAt('evnt'),
|
|
682
|
-
getId: (e) => e.eventId,
|
|
683
|
-
});
|
|
684
|
-
events = allEvents.data;
|
|
685
|
-
}
|
|
686
|
-
// Return EventResult with event and any created/updated entity
|
|
687
|
-
return {
|
|
688
|
-
event: filteredEvent,
|
|
689
|
-
run,
|
|
690
|
-
step,
|
|
691
|
-
hook,
|
|
692
|
-
wait,
|
|
693
|
-
events,
|
|
694
|
-
};
|
|
772
|
+
} // end createImpl
|
|
695
773
|
},
|
|
696
774
|
async get(runId, eventId, params) {
|
|
697
775
|
assertSafeEntityId('runId', runId);
|