@pgflow/core 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# @pgflow/core
|
|
2
2
|
|
|
3
|
+
## 0.7.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- @pgflow/dsl@0.7.3
|
|
8
|
+
|
|
9
|
+
## 0.7.2
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- c22a1e5: Fix missing realtime broadcasts for step:started and step:completed events
|
|
14
|
+
|
|
15
|
+
**Critical bug fix:** Clients were not receiving `step:started` events when steps transitioned to Started status, and `step:completed` events for empty map steps and cascade completions were also missing.
|
|
16
|
+
|
|
17
|
+
**Root cause:** PostgreSQL query optimizer was eliminating CTEs containing `realtime.send()` calls because they were not referenced by subsequent operations or the final RETURN statement.
|
|
18
|
+
|
|
19
|
+
**Solution:** Moved `realtime.send()` calls directly into RETURNING clauses of UPDATE statements, ensuring they execute atomically with state changes and cannot be optimized away.
|
|
20
|
+
|
|
21
|
+
**Changes:**
|
|
22
|
+
|
|
23
|
+
- `start_ready_steps()`: Broadcasts step:started and step:completed events in RETURNING clauses
|
|
24
|
+
- `cascade_complete_taskless_steps()`: Broadcasts step:completed events atomically with cascade completion
|
|
25
|
+
- `complete_task()`: Added PERFORM statements for run:failed and step:failed broadcasts
|
|
26
|
+
- Client: Added `applySnapshot()` methods to FlowRun and FlowStep for proper initial state hydration without event emission
|
|
27
|
+
- @pgflow/dsl@0.7.2
|
|
28
|
+
|
|
3
29
|
## 0.7.1
|
|
4
30
|
|
|
5
31
|
### Patch Changes
|
package/dist/package.json
CHANGED
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
-- Modify "cascade_complete_taskless_steps" function
|
|
2
|
+
CREATE OR REPLACE FUNCTION "pgflow"."cascade_complete_taskless_steps" ("run_id" uuid) RETURNS integer LANGUAGE plpgsql AS $$
|
|
3
|
+
DECLARE
|
|
4
|
+
v_total_completed int := 0;
|
|
5
|
+
v_iteration_completed int;
|
|
6
|
+
v_iterations int := 0;
|
|
7
|
+
v_max_iterations int := 50;
|
|
8
|
+
BEGIN
|
|
9
|
+
-- ==========================================
|
|
10
|
+
-- ITERATIVE CASCADE COMPLETION
|
|
11
|
+
-- ==========================================
|
|
12
|
+
-- Completes taskless steps in waves until none remain
|
|
13
|
+
LOOP
|
|
14
|
+
-- ---------- Safety check ----------
|
|
15
|
+
v_iterations := v_iterations + 1;
|
|
16
|
+
IF v_iterations > v_max_iterations THEN
|
|
17
|
+
RAISE EXCEPTION 'Cascade loop exceeded safety limit of % iterations', v_max_iterations;
|
|
18
|
+
END IF;
|
|
19
|
+
|
|
20
|
+
-- ==========================================
|
|
21
|
+
-- COMPLETE READY TASKLESS STEPS
|
|
22
|
+
-- ==========================================
|
|
23
|
+
WITH
|
|
24
|
+
-- ---------- Find steps to complete in topological order ----------
|
|
25
|
+
steps_to_complete AS (
|
|
26
|
+
SELECT ss.run_id, ss.step_slug
|
|
27
|
+
FROM pgflow.step_states ss
|
|
28
|
+
JOIN pgflow.steps s ON s.flow_slug = ss.flow_slug AND s.step_slug = ss.step_slug
|
|
29
|
+
WHERE ss.run_id = cascade_complete_taskless_steps.run_id
|
|
30
|
+
AND ss.status = 'created'
|
|
31
|
+
AND ss.remaining_deps = 0
|
|
32
|
+
AND ss.initial_tasks = 0
|
|
33
|
+
-- Process in topological order to ensure proper cascade
|
|
34
|
+
ORDER BY s.step_index
|
|
35
|
+
),
|
|
36
|
+
completed AS (
|
|
37
|
+
-- ---------- Complete taskless steps ----------
|
|
38
|
+
-- Steps with initial_tasks=0 and no remaining deps
|
|
39
|
+
UPDATE pgflow.step_states ss
|
|
40
|
+
SET status = 'completed',
|
|
41
|
+
started_at = now(),
|
|
42
|
+
completed_at = now(),
|
|
43
|
+
remaining_tasks = 0
|
|
44
|
+
FROM steps_to_complete stc
|
|
45
|
+
WHERE ss.run_id = stc.run_id
|
|
46
|
+
AND ss.step_slug = stc.step_slug
|
|
47
|
+
RETURNING
|
|
48
|
+
ss.*,
|
|
49
|
+
-- Broadcast step:completed event atomically with the UPDATE
|
|
50
|
+
-- Using RETURNING ensures this executes during row processing
|
|
51
|
+
-- and cannot be optimized away by the query planner
|
|
52
|
+
realtime.send(
|
|
53
|
+
jsonb_build_object(
|
|
54
|
+
'event_type', 'step:completed',
|
|
55
|
+
'run_id', ss.run_id,
|
|
56
|
+
'step_slug', ss.step_slug,
|
|
57
|
+
'status', 'completed',
|
|
58
|
+
'started_at', ss.started_at,
|
|
59
|
+
'completed_at', ss.completed_at,
|
|
60
|
+
'remaining_tasks', 0,
|
|
61
|
+
'remaining_deps', 0,
|
|
62
|
+
'output', '[]'::jsonb
|
|
63
|
+
),
|
|
64
|
+
concat('step:', ss.step_slug, ':completed'),
|
|
65
|
+
concat('pgflow:run:', ss.run_id),
|
|
66
|
+
false
|
|
67
|
+
) as _broadcast_result -- Prefix with _ to indicate internal use only
|
|
68
|
+
),
|
|
69
|
+
-- ---------- Update dependent steps ----------
|
|
70
|
+
-- Propagate completion and empty arrays to dependents
|
|
71
|
+
dep_updates AS (
|
|
72
|
+
UPDATE pgflow.step_states ss
|
|
73
|
+
SET remaining_deps = ss.remaining_deps - dep_count.count,
|
|
74
|
+
-- If the dependent is a map step and its dependency completed with 0 tasks,
|
|
75
|
+
-- set its initial_tasks to 0 as well
|
|
76
|
+
initial_tasks = CASE
|
|
77
|
+
WHEN s.step_type = 'map' AND dep_count.has_zero_tasks
|
|
78
|
+
THEN 0 -- Empty array propagation
|
|
79
|
+
ELSE ss.initial_tasks -- Keep existing value (including NULL)
|
|
80
|
+
END
|
|
81
|
+
FROM (
|
|
82
|
+
-- Aggregate dependency updates per dependent step
|
|
83
|
+
SELECT
|
|
84
|
+
d.flow_slug,
|
|
85
|
+
d.step_slug as dependent_slug,
|
|
86
|
+
COUNT(*) as count,
|
|
87
|
+
BOOL_OR(c.initial_tasks = 0) as has_zero_tasks
|
|
88
|
+
FROM completed c
|
|
89
|
+
JOIN pgflow.deps d ON d.flow_slug = c.flow_slug
|
|
90
|
+
AND d.dep_slug = c.step_slug
|
|
91
|
+
GROUP BY d.flow_slug, d.step_slug
|
|
92
|
+
) dep_count,
|
|
93
|
+
pgflow.steps s
|
|
94
|
+
WHERE ss.run_id = cascade_complete_taskless_steps.run_id
|
|
95
|
+
AND ss.flow_slug = dep_count.flow_slug
|
|
96
|
+
AND ss.step_slug = dep_count.dependent_slug
|
|
97
|
+
AND s.flow_slug = ss.flow_slug
|
|
98
|
+
AND s.step_slug = ss.step_slug
|
|
99
|
+
),
|
|
100
|
+
-- ---------- Update run counters ----------
|
|
101
|
+
-- Only decrement remaining_steps; let maybe_complete_run handle finalization
|
|
102
|
+
run_updates AS (
|
|
103
|
+
UPDATE pgflow.runs r
|
|
104
|
+
SET remaining_steps = r.remaining_steps - c.completed_count
|
|
105
|
+
FROM (SELECT COUNT(*) AS completed_count FROM completed) c
|
|
106
|
+
WHERE r.run_id = cascade_complete_taskless_steps.run_id
|
|
107
|
+
AND c.completed_count > 0
|
|
108
|
+
)
|
|
109
|
+
-- ---------- Check iteration results ----------
|
|
110
|
+
SELECT COUNT(*) INTO v_iteration_completed FROM completed;
|
|
111
|
+
|
|
112
|
+
EXIT WHEN v_iteration_completed = 0; -- No more steps to complete
|
|
113
|
+
v_total_completed := v_total_completed + v_iteration_completed;
|
|
114
|
+
END LOOP;
|
|
115
|
+
|
|
116
|
+
RETURN v_total_completed;
|
|
117
|
+
END;
|
|
118
|
+
$$;
|
|
119
|
+
-- Modify "start_ready_steps" function
|
|
120
|
+
CREATE OR REPLACE FUNCTION "pgflow"."start_ready_steps" ("run_id" uuid) RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
121
|
+
begin
|
|
122
|
+
-- ==========================================
|
|
123
|
+
-- GUARD: No mutations on failed runs
|
|
124
|
+
-- ==========================================
|
|
125
|
+
IF EXISTS (SELECT 1 FROM pgflow.runs WHERE pgflow.runs.run_id = start_ready_steps.run_id AND pgflow.runs.status = 'failed') THEN
|
|
126
|
+
RETURN;
|
|
127
|
+
END IF;
|
|
128
|
+
|
|
129
|
+
-- ==========================================
|
|
130
|
+
-- HANDLE EMPTY ARRAY MAPS (initial_tasks = 0)
|
|
131
|
+
-- ==========================================
|
|
132
|
+
-- These complete immediately without spawning tasks
|
|
133
|
+
WITH empty_map_steps AS (
|
|
134
|
+
SELECT step_state.*
|
|
135
|
+
FROM pgflow.step_states AS step_state
|
|
136
|
+
JOIN pgflow.steps AS step
|
|
137
|
+
ON step.flow_slug = step_state.flow_slug
|
|
138
|
+
AND step.step_slug = step_state.step_slug
|
|
139
|
+
WHERE step_state.run_id = start_ready_steps.run_id
|
|
140
|
+
AND step_state.status = 'created'
|
|
141
|
+
AND step_state.remaining_deps = 0
|
|
142
|
+
AND step.step_type = 'map'
|
|
143
|
+
AND step_state.initial_tasks = 0
|
|
144
|
+
ORDER BY step_state.step_slug
|
|
145
|
+
FOR UPDATE OF step_state
|
|
146
|
+
),
|
|
147
|
+
-- ---------- Complete empty map steps ----------
|
|
148
|
+
completed_empty_steps AS (
|
|
149
|
+
UPDATE pgflow.step_states
|
|
150
|
+
SET status = 'completed',
|
|
151
|
+
started_at = now(),
|
|
152
|
+
completed_at = now(),
|
|
153
|
+
remaining_tasks = 0
|
|
154
|
+
FROM empty_map_steps
|
|
155
|
+
WHERE pgflow.step_states.run_id = start_ready_steps.run_id
|
|
156
|
+
AND pgflow.step_states.step_slug = empty_map_steps.step_slug
|
|
157
|
+
RETURNING
|
|
158
|
+
pgflow.step_states.*,
|
|
159
|
+
-- Broadcast step:completed event atomically with the UPDATE
|
|
160
|
+
-- Using RETURNING ensures this executes during row processing
|
|
161
|
+
-- and cannot be optimized away by the query planner
|
|
162
|
+
realtime.send(
|
|
163
|
+
jsonb_build_object(
|
|
164
|
+
'event_type', 'step:completed',
|
|
165
|
+
'run_id', pgflow.step_states.run_id,
|
|
166
|
+
'step_slug', pgflow.step_states.step_slug,
|
|
167
|
+
'status', 'completed',
|
|
168
|
+
'started_at', pgflow.step_states.started_at,
|
|
169
|
+
'completed_at', pgflow.step_states.completed_at,
|
|
170
|
+
'remaining_tasks', 0,
|
|
171
|
+
'remaining_deps', 0,
|
|
172
|
+
'output', '[]'::jsonb
|
|
173
|
+
),
|
|
174
|
+
concat('step:', pgflow.step_states.step_slug, ':completed'),
|
|
175
|
+
concat('pgflow:run:', pgflow.step_states.run_id),
|
|
176
|
+
false
|
|
177
|
+
) as _broadcast_completed -- Prefix with _ to indicate internal use only
|
|
178
|
+
),
|
|
179
|
+
|
|
180
|
+
-- ==========================================
|
|
181
|
+
-- HANDLE NORMAL STEPS (initial_tasks > 0)
|
|
182
|
+
-- ==========================================
|
|
183
|
+
-- ---------- Find ready steps ----------
|
|
184
|
+
-- Steps with no remaining deps and known task count
|
|
185
|
+
ready_steps AS (
|
|
186
|
+
SELECT *
|
|
187
|
+
FROM pgflow.step_states AS step_state
|
|
188
|
+
WHERE step_state.run_id = start_ready_steps.run_id
|
|
189
|
+
AND step_state.status = 'created'
|
|
190
|
+
AND step_state.remaining_deps = 0
|
|
191
|
+
AND step_state.initial_tasks IS NOT NULL -- NEW: Cannot start with unknown count
|
|
192
|
+
AND step_state.initial_tasks > 0 -- Don't start taskless steps
|
|
193
|
+
-- Exclude empty map steps already handled
|
|
194
|
+
AND NOT EXISTS (
|
|
195
|
+
SELECT 1 FROM empty_map_steps
|
|
196
|
+
WHERE empty_map_steps.run_id = step_state.run_id
|
|
197
|
+
AND empty_map_steps.step_slug = step_state.step_slug
|
|
198
|
+
)
|
|
199
|
+
ORDER BY step_state.step_slug
|
|
200
|
+
FOR UPDATE
|
|
201
|
+
),
|
|
202
|
+
-- ---------- Mark steps as started ----------
|
|
203
|
+
started_step_states AS (
|
|
204
|
+
UPDATE pgflow.step_states
|
|
205
|
+
SET status = 'started',
|
|
206
|
+
started_at = now(),
|
|
207
|
+
remaining_tasks = ready_steps.initial_tasks -- Copy initial_tasks to remaining_tasks when starting
|
|
208
|
+
FROM ready_steps
|
|
209
|
+
WHERE pgflow.step_states.run_id = start_ready_steps.run_id
|
|
210
|
+
AND pgflow.step_states.step_slug = ready_steps.step_slug
|
|
211
|
+
RETURNING pgflow.step_states.*,
|
|
212
|
+
-- Broadcast step:started event atomically with the UPDATE
|
|
213
|
+
-- Using RETURNING ensures this executes during row processing
|
|
214
|
+
-- and cannot be optimized away by the query planner
|
|
215
|
+
realtime.send(
|
|
216
|
+
jsonb_build_object(
|
|
217
|
+
'event_type', 'step:started',
|
|
218
|
+
'run_id', pgflow.step_states.run_id,
|
|
219
|
+
'step_slug', pgflow.step_states.step_slug,
|
|
220
|
+
'status', 'started',
|
|
221
|
+
'started_at', pgflow.step_states.started_at,
|
|
222
|
+
'remaining_tasks', pgflow.step_states.remaining_tasks,
|
|
223
|
+
'remaining_deps', pgflow.step_states.remaining_deps
|
|
224
|
+
),
|
|
225
|
+
concat('step:', pgflow.step_states.step_slug, ':started'),
|
|
226
|
+
concat('pgflow:run:', pgflow.step_states.run_id),
|
|
227
|
+
false
|
|
228
|
+
) as _broadcast_result -- Prefix with _ to indicate internal use only
|
|
229
|
+
),
|
|
230
|
+
|
|
231
|
+
-- ==========================================
|
|
232
|
+
-- TASK GENERATION AND QUEUE MESSAGES
|
|
233
|
+
-- ==========================================
|
|
234
|
+
-- ---------- Generate tasks and batch messages ----------
|
|
235
|
+
-- Single steps: 1 task (index 0)
|
|
236
|
+
-- Map steps: N tasks (indices 0..N-1)
|
|
237
|
+
message_batches AS (
|
|
238
|
+
SELECT
|
|
239
|
+
started_step.flow_slug,
|
|
240
|
+
started_step.run_id,
|
|
241
|
+
started_step.step_slug,
|
|
242
|
+
COALESCE(step.opt_start_delay, 0) as delay,
|
|
243
|
+
array_agg(
|
|
244
|
+
jsonb_build_object(
|
|
245
|
+
'flow_slug', started_step.flow_slug,
|
|
246
|
+
'run_id', started_step.run_id,
|
|
247
|
+
'step_slug', started_step.step_slug,
|
|
248
|
+
'task_index', task_idx.task_index
|
|
249
|
+
) ORDER BY task_idx.task_index
|
|
250
|
+
) AS messages,
|
|
251
|
+
array_agg(task_idx.task_index ORDER BY task_idx.task_index) AS task_indices
|
|
252
|
+
FROM started_step_states AS started_step
|
|
253
|
+
JOIN pgflow.steps AS step
|
|
254
|
+
ON step.flow_slug = started_step.flow_slug
|
|
255
|
+
AND step.step_slug = started_step.step_slug
|
|
256
|
+
-- Generate task indices from 0 to initial_tasks-1
|
|
257
|
+
CROSS JOIN LATERAL generate_series(0, started_step.initial_tasks - 1) AS task_idx(task_index)
|
|
258
|
+
GROUP BY started_step.flow_slug, started_step.run_id, started_step.step_slug, step.opt_start_delay
|
|
259
|
+
),
|
|
260
|
+
-- ---------- Send messages to queue ----------
|
|
261
|
+
-- Uses batch sending for performance with large arrays
|
|
262
|
+
sent_messages AS (
|
|
263
|
+
SELECT
|
|
264
|
+
mb.flow_slug,
|
|
265
|
+
mb.run_id,
|
|
266
|
+
mb.step_slug,
|
|
267
|
+
task_indices.task_index,
|
|
268
|
+
msg_ids.msg_id
|
|
269
|
+
FROM message_batches mb
|
|
270
|
+
CROSS JOIN LATERAL unnest(mb.task_indices) WITH ORDINALITY AS task_indices(task_index, idx_ord)
|
|
271
|
+
CROSS JOIN LATERAL pgmq.send_batch(mb.flow_slug, mb.messages, mb.delay) WITH ORDINALITY AS msg_ids(msg_id, msg_ord)
|
|
272
|
+
WHERE task_indices.idx_ord = msg_ids.msg_ord
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
-- ==========================================
|
|
276
|
+
-- RECORD TASKS IN DATABASE
|
|
277
|
+
-- ==========================================
|
|
278
|
+
INSERT INTO pgflow.step_tasks (flow_slug, run_id, step_slug, task_index, message_id)
|
|
279
|
+
SELECT
|
|
280
|
+
sent_messages.flow_slug,
|
|
281
|
+
sent_messages.run_id,
|
|
282
|
+
sent_messages.step_slug,
|
|
283
|
+
sent_messages.task_index,
|
|
284
|
+
sent_messages.msg_id
|
|
285
|
+
FROM sent_messages;
|
|
286
|
+
|
|
287
|
+
-- ==========================================
|
|
288
|
+
-- BROADCAST REALTIME EVENTS
|
|
289
|
+
-- ==========================================
|
|
290
|
+
-- Note: Both step:completed events for empty maps and step:started events
|
|
291
|
+
-- are now broadcast atomically in their respective CTEs using RETURNING pattern.
|
|
292
|
+
-- This ensures correct ordering, prevents duplicate broadcasts, and guarantees
|
|
293
|
+
-- that events are sent for exactly the rows that were updated.
|
|
294
|
+
|
|
295
|
+
end;
|
|
296
|
+
$$;
|
|
297
|
+
-- Modify "complete_task" function
|
|
298
|
+
CREATE OR REPLACE FUNCTION "pgflow"."complete_task" ("run_id" uuid, "step_slug" text, "task_index" integer, "output" jsonb) RETURNS SETOF "pgflow"."step_tasks" LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
299
|
+
declare
|
|
300
|
+
v_step_state pgflow.step_states%ROWTYPE;
|
|
301
|
+
v_dependent_map_slug text;
|
|
302
|
+
v_run_record pgflow.runs%ROWTYPE;
|
|
303
|
+
v_step_record pgflow.step_states%ROWTYPE;
|
|
304
|
+
begin
|
|
305
|
+
|
|
306
|
+
-- ==========================================
|
|
307
|
+
-- GUARD: No mutations on failed runs
|
|
308
|
+
-- ==========================================
|
|
309
|
+
IF EXISTS (SELECT 1 FROM pgflow.runs WHERE pgflow.runs.run_id = complete_task.run_id AND pgflow.runs.status = 'failed') THEN
|
|
310
|
+
RETURN QUERY SELECT * FROM pgflow.step_tasks
|
|
311
|
+
WHERE pgflow.step_tasks.run_id = complete_task.run_id
|
|
312
|
+
AND pgflow.step_tasks.step_slug = complete_task.step_slug
|
|
313
|
+
AND pgflow.step_tasks.task_index = complete_task.task_index;
|
|
314
|
+
RETURN;
|
|
315
|
+
END IF;
|
|
316
|
+
|
|
317
|
+
-- ==========================================
|
|
318
|
+
-- LOCK ACQUISITION AND TYPE VALIDATION
|
|
319
|
+
-- ==========================================
|
|
320
|
+
-- Acquire locks first to prevent race conditions
|
|
321
|
+
SELECT * INTO v_run_record FROM pgflow.runs
|
|
322
|
+
WHERE pgflow.runs.run_id = complete_task.run_id
|
|
323
|
+
FOR UPDATE;
|
|
324
|
+
|
|
325
|
+
SELECT * INTO v_step_record FROM pgflow.step_states
|
|
326
|
+
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
327
|
+
AND pgflow.step_states.step_slug = complete_task.step_slug
|
|
328
|
+
FOR UPDATE;
|
|
329
|
+
|
|
330
|
+
-- Check for type violations AFTER acquiring locks
|
|
331
|
+
SELECT child_step.step_slug INTO v_dependent_map_slug
|
|
332
|
+
FROM pgflow.deps dependency
|
|
333
|
+
JOIN pgflow.steps child_step ON child_step.flow_slug = dependency.flow_slug
|
|
334
|
+
AND child_step.step_slug = dependency.step_slug
|
|
335
|
+
JOIN pgflow.steps parent_step ON parent_step.flow_slug = dependency.flow_slug
|
|
336
|
+
AND parent_step.step_slug = dependency.dep_slug
|
|
337
|
+
JOIN pgflow.step_states child_state ON child_state.flow_slug = child_step.flow_slug
|
|
338
|
+
AND child_state.step_slug = child_step.step_slug
|
|
339
|
+
WHERE dependency.dep_slug = complete_task.step_slug -- parent is the completing step
|
|
340
|
+
AND dependency.flow_slug = v_run_record.flow_slug
|
|
341
|
+
AND parent_step.step_type = 'single' -- Only validate single steps
|
|
342
|
+
AND child_step.step_type = 'map'
|
|
343
|
+
AND child_state.run_id = complete_task.run_id
|
|
344
|
+
AND child_state.initial_tasks IS NULL
|
|
345
|
+
AND (complete_task.output IS NULL OR jsonb_typeof(complete_task.output) != 'array')
|
|
346
|
+
LIMIT 1;
|
|
347
|
+
|
|
348
|
+
-- Handle type violation if detected
|
|
349
|
+
IF v_dependent_map_slug IS NOT NULL THEN
|
|
350
|
+
-- Mark run as failed immediately
|
|
351
|
+
UPDATE pgflow.runs
|
|
352
|
+
SET status = 'failed',
|
|
353
|
+
failed_at = now()
|
|
354
|
+
WHERE pgflow.runs.run_id = complete_task.run_id;
|
|
355
|
+
|
|
356
|
+
-- Broadcast run:failed event
|
|
357
|
+
-- Uses PERFORM pattern to ensure execution (proven reliable pattern in this function)
|
|
358
|
+
PERFORM realtime.send(
|
|
359
|
+
jsonb_build_object(
|
|
360
|
+
'event_type', 'run:failed',
|
|
361
|
+
'run_id', complete_task.run_id,
|
|
362
|
+
'flow_slug', v_run_record.flow_slug,
|
|
363
|
+
'status', 'failed',
|
|
364
|
+
'failed_at', now()
|
|
365
|
+
),
|
|
366
|
+
'run:failed',
|
|
367
|
+
concat('pgflow:run:', complete_task.run_id),
|
|
368
|
+
false
|
|
369
|
+
);
|
|
370
|
+
|
|
371
|
+
-- Archive all active messages (both queued and started) to prevent orphaned messages
|
|
372
|
+
PERFORM pgmq.archive(
|
|
373
|
+
v_run_record.flow_slug,
|
|
374
|
+
array_agg(st.message_id)
|
|
375
|
+
)
|
|
376
|
+
FROM pgflow.step_tasks st
|
|
377
|
+
WHERE st.run_id = complete_task.run_id
|
|
378
|
+
AND st.status IN ('queued', 'started')
|
|
379
|
+
AND st.message_id IS NOT NULL
|
|
380
|
+
HAVING count(*) > 0; -- Only call archive if there are messages to archive
|
|
381
|
+
|
|
382
|
+
-- Mark current task as failed and store the output
|
|
383
|
+
UPDATE pgflow.step_tasks
|
|
384
|
+
SET status = 'failed',
|
|
385
|
+
failed_at = now(),
|
|
386
|
+
output = complete_task.output, -- Store the output that caused the violation
|
|
387
|
+
error_message = '[TYPE_VIOLATION] Produced ' ||
|
|
388
|
+
CASE WHEN complete_task.output IS NULL THEN 'null'
|
|
389
|
+
ELSE jsonb_typeof(complete_task.output) END ||
|
|
390
|
+
' instead of array'
|
|
391
|
+
WHERE pgflow.step_tasks.run_id = complete_task.run_id
|
|
392
|
+
AND pgflow.step_tasks.step_slug = complete_task.step_slug
|
|
393
|
+
AND pgflow.step_tasks.task_index = complete_task.task_index;
|
|
394
|
+
|
|
395
|
+
-- Mark step state as failed
|
|
396
|
+
UPDATE pgflow.step_states
|
|
397
|
+
SET status = 'failed',
|
|
398
|
+
failed_at = now(),
|
|
399
|
+
error_message = '[TYPE_VIOLATION] Map step ' || v_dependent_map_slug ||
|
|
400
|
+
' expects array input but dependency ' || complete_task.step_slug ||
|
|
401
|
+
' produced ' || CASE WHEN complete_task.output IS NULL THEN 'null'
|
|
402
|
+
ELSE jsonb_typeof(complete_task.output) END
|
|
403
|
+
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
404
|
+
AND pgflow.step_states.step_slug = complete_task.step_slug;
|
|
405
|
+
|
|
406
|
+
-- Broadcast step:failed event
|
|
407
|
+
-- Uses PERFORM pattern to ensure execution (proven reliable pattern in this function)
|
|
408
|
+
PERFORM realtime.send(
|
|
409
|
+
jsonb_build_object(
|
|
410
|
+
'event_type', 'step:failed',
|
|
411
|
+
'run_id', complete_task.run_id,
|
|
412
|
+
'step_slug', complete_task.step_slug,
|
|
413
|
+
'status', 'failed',
|
|
414
|
+
'error_message', '[TYPE_VIOLATION] Map step ' || v_dependent_map_slug ||
|
|
415
|
+
' expects array input but dependency ' || complete_task.step_slug ||
|
|
416
|
+
' produced ' || CASE WHEN complete_task.output IS NULL THEN 'null'
|
|
417
|
+
ELSE jsonb_typeof(complete_task.output) END,
|
|
418
|
+
'failed_at', now()
|
|
419
|
+
),
|
|
420
|
+
concat('step:', complete_task.step_slug, ':failed'),
|
|
421
|
+
concat('pgflow:run:', complete_task.run_id),
|
|
422
|
+
false
|
|
423
|
+
);
|
|
424
|
+
|
|
425
|
+
-- Archive the current task's message (it was started, now failed)
|
|
426
|
+
PERFORM pgmq.archive(
|
|
427
|
+
v_run_record.flow_slug,
|
|
428
|
+
st.message_id -- Single message, use scalar form
|
|
429
|
+
)
|
|
430
|
+
FROM pgflow.step_tasks st
|
|
431
|
+
WHERE st.run_id = complete_task.run_id
|
|
432
|
+
AND st.step_slug = complete_task.step_slug
|
|
433
|
+
AND st.task_index = complete_task.task_index
|
|
434
|
+
AND st.message_id IS NOT NULL;
|
|
435
|
+
|
|
436
|
+
-- Return empty result
|
|
437
|
+
RETURN QUERY SELECT * FROM pgflow.step_tasks WHERE false;
|
|
438
|
+
RETURN;
|
|
439
|
+
END IF;
|
|
440
|
+
|
|
441
|
+
-- ==========================================
|
|
442
|
+
-- MAIN CTE CHAIN: Update task and propagate changes
|
|
443
|
+
-- ==========================================
|
|
444
|
+
WITH
|
|
445
|
+
-- ---------- Task completion ----------
|
|
446
|
+
-- Update the task record with completion status and output
|
|
447
|
+
task AS (
|
|
448
|
+
UPDATE pgflow.step_tasks
|
|
449
|
+
SET
|
|
450
|
+
status = 'completed',
|
|
451
|
+
completed_at = now(),
|
|
452
|
+
output = complete_task.output
|
|
453
|
+
WHERE pgflow.step_tasks.run_id = complete_task.run_id
|
|
454
|
+
AND pgflow.step_tasks.step_slug = complete_task.step_slug
|
|
455
|
+
AND pgflow.step_tasks.task_index = complete_task.task_index
|
|
456
|
+
AND pgflow.step_tasks.status = 'started'
|
|
457
|
+
RETURNING *
|
|
458
|
+
),
|
|
459
|
+
-- ---------- Step state update ----------
|
|
460
|
+
-- Decrement remaining_tasks and potentially mark step as completed
|
|
461
|
+
step_state AS (
|
|
462
|
+
UPDATE pgflow.step_states
|
|
463
|
+
SET
|
|
464
|
+
status = CASE
|
|
465
|
+
WHEN pgflow.step_states.remaining_tasks = 1 THEN 'completed' -- Will be 0 after decrement
|
|
466
|
+
ELSE 'started'
|
|
467
|
+
END,
|
|
468
|
+
completed_at = CASE
|
|
469
|
+
WHEN pgflow.step_states.remaining_tasks = 1 THEN now() -- Will be 0 after decrement
|
|
470
|
+
ELSE NULL
|
|
471
|
+
END,
|
|
472
|
+
remaining_tasks = pgflow.step_states.remaining_tasks - 1
|
|
473
|
+
FROM task
|
|
474
|
+
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
475
|
+
AND pgflow.step_states.step_slug = complete_task.step_slug
|
|
476
|
+
RETURNING pgflow.step_states.*
|
|
477
|
+
),
|
|
478
|
+
-- ---------- Dependency resolution ----------
|
|
479
|
+
-- Find all child steps that depend on the completed parent step (only if parent completed)
|
|
480
|
+
child_steps AS (
|
|
481
|
+
SELECT deps.step_slug AS child_step_slug
|
|
482
|
+
FROM pgflow.deps deps
|
|
483
|
+
JOIN step_state parent_state ON parent_state.status = 'completed' AND deps.flow_slug = parent_state.flow_slug
|
|
484
|
+
WHERE deps.dep_slug = complete_task.step_slug -- dep_slug is the parent, step_slug is the child
|
|
485
|
+
ORDER BY deps.step_slug -- Ensure consistent ordering
|
|
486
|
+
),
|
|
487
|
+
-- ---------- Lock child steps ----------
|
|
488
|
+
-- Acquire locks on all child steps before updating them
|
|
489
|
+
child_steps_lock AS (
|
|
490
|
+
SELECT * FROM pgflow.step_states
|
|
491
|
+
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
492
|
+
AND pgflow.step_states.step_slug IN (SELECT child_step_slug FROM child_steps)
|
|
493
|
+
FOR UPDATE
|
|
494
|
+
),
|
|
495
|
+
-- ---------- Update child steps ----------
|
|
496
|
+
-- Decrement remaining_deps and resolve NULL initial_tasks for map steps
|
|
497
|
+
child_steps_update AS (
|
|
498
|
+
UPDATE pgflow.step_states child_state
|
|
499
|
+
SET remaining_deps = child_state.remaining_deps - 1,
|
|
500
|
+
-- Resolve NULL initial_tasks for child map steps
|
|
501
|
+
-- This is where child maps learn their array size from the parent
|
|
502
|
+
-- This CTE only runs when the parent step is complete (see child_steps JOIN)
|
|
503
|
+
initial_tasks = CASE
|
|
504
|
+
WHEN child_step.step_type = 'map' AND child_state.initial_tasks IS NULL THEN
|
|
505
|
+
CASE
|
|
506
|
+
WHEN parent_step.step_type = 'map' THEN
|
|
507
|
+
-- Map->map: Count all completed tasks from parent map
|
|
508
|
+
-- We add 1 because the current task is being completed in this transaction
|
|
509
|
+
-- but isn't yet visible as 'completed' in the step_tasks table
|
|
510
|
+
-- TODO: Refactor to use future column step_states.total_tasks
|
|
511
|
+
-- Would eliminate the COUNT query and just use parent_state.total_tasks
|
|
512
|
+
(SELECT COUNT(*)::int + 1
|
|
513
|
+
FROM pgflow.step_tasks parent_tasks
|
|
514
|
+
WHERE parent_tasks.run_id = complete_task.run_id
|
|
515
|
+
AND parent_tasks.step_slug = complete_task.step_slug
|
|
516
|
+
AND parent_tasks.status = 'completed'
|
|
517
|
+
AND parent_tasks.task_index != complete_task.task_index)
|
|
518
|
+
ELSE
|
|
519
|
+
-- Single->map: Use output array length (single steps complete immediately)
|
|
520
|
+
CASE
|
|
521
|
+
WHEN complete_task.output IS NOT NULL
|
|
522
|
+
AND jsonb_typeof(complete_task.output) = 'array' THEN
|
|
523
|
+
jsonb_array_length(complete_task.output)
|
|
524
|
+
ELSE NULL -- Keep NULL if not an array
|
|
525
|
+
END
|
|
526
|
+
END
|
|
527
|
+
ELSE child_state.initial_tasks -- Keep existing value (including NULL)
|
|
528
|
+
END
|
|
529
|
+
FROM child_steps children
|
|
530
|
+
JOIN pgflow.steps child_step ON child_step.flow_slug = (SELECT r.flow_slug FROM pgflow.runs r WHERE r.run_id = complete_task.run_id)
|
|
531
|
+
AND child_step.step_slug = children.child_step_slug
|
|
532
|
+
JOIN pgflow.steps parent_step ON parent_step.flow_slug = (SELECT r.flow_slug FROM pgflow.runs r WHERE r.run_id = complete_task.run_id)
|
|
533
|
+
AND parent_step.step_slug = complete_task.step_slug
|
|
534
|
+
WHERE child_state.run_id = complete_task.run_id
|
|
535
|
+
AND child_state.step_slug = children.child_step_slug
|
|
536
|
+
)
|
|
537
|
+
-- ---------- Update run remaining_steps ----------
|
|
538
|
+
-- Decrement the run's remaining_steps counter if step completed
|
|
539
|
+
UPDATE pgflow.runs
|
|
540
|
+
SET remaining_steps = pgflow.runs.remaining_steps - 1
|
|
541
|
+
FROM step_state
|
|
542
|
+
WHERE pgflow.runs.run_id = complete_task.run_id
|
|
543
|
+
AND step_state.status = 'completed';
|
|
544
|
+
|
|
545
|
+
-- ==========================================
|
|
546
|
+
-- POST-COMPLETION ACTIONS
|
|
547
|
+
-- ==========================================
|
|
548
|
+
|
|
549
|
+
-- ---------- Get updated state for broadcasting ----------
|
|
550
|
+
SELECT * INTO v_step_state FROM pgflow.step_states
|
|
551
|
+
WHERE pgflow.step_states.run_id = complete_task.run_id AND pgflow.step_states.step_slug = complete_task.step_slug;
|
|
552
|
+
|
|
553
|
+
-- ---------- Handle step completion ----------
|
|
554
|
+
IF v_step_state.status = 'completed' THEN
|
|
555
|
+
-- Broadcast step:completed event FIRST (before cascade)
|
|
556
|
+
-- This ensures parent broadcasts before its dependent children
|
|
557
|
+
-- For map steps, aggregate all task outputs; for single steps, use the task output
|
|
558
|
+
PERFORM realtime.send(
|
|
559
|
+
jsonb_build_object(
|
|
560
|
+
'event_type', 'step:completed',
|
|
561
|
+
'run_id', complete_task.run_id,
|
|
562
|
+
'step_slug', complete_task.step_slug,
|
|
563
|
+
'status', 'completed',
|
|
564
|
+
'output', CASE
|
|
565
|
+
WHEN (SELECT s.step_type FROM pgflow.steps s
|
|
566
|
+
WHERE s.flow_slug = v_step_state.flow_slug
|
|
567
|
+
AND s.step_slug = complete_task.step_slug) = 'map' THEN
|
|
568
|
+
-- Aggregate all task outputs for map steps
|
|
569
|
+
(SELECT COALESCE(jsonb_agg(st.output ORDER BY st.task_index), '[]'::jsonb)
|
|
570
|
+
FROM pgflow.step_tasks st
|
|
571
|
+
WHERE st.run_id = complete_task.run_id
|
|
572
|
+
AND st.step_slug = complete_task.step_slug
|
|
573
|
+
AND st.status = 'completed')
|
|
574
|
+
ELSE
|
|
575
|
+
-- Single step: use the individual task output
|
|
576
|
+
complete_task.output
|
|
577
|
+
END,
|
|
578
|
+
'completed_at', v_step_state.completed_at
|
|
579
|
+
),
|
|
580
|
+
concat('step:', complete_task.step_slug, ':completed'),
|
|
581
|
+
concat('pgflow:run:', complete_task.run_id),
|
|
582
|
+
false
|
|
583
|
+
);
|
|
584
|
+
|
|
585
|
+
-- THEN cascade complete any taskless steps that are now ready
|
|
586
|
+
-- This ensures dependent children broadcast AFTER their parent
|
|
587
|
+
PERFORM pgflow.cascade_complete_taskless_steps(complete_task.run_id);
|
|
588
|
+
END IF;
|
|
589
|
+
|
|
590
|
+
-- ---------- Archive completed task message ----------
|
|
591
|
+
-- Move message from active queue to archive table
|
|
592
|
+
PERFORM (
|
|
593
|
+
WITH completed_tasks AS (
|
|
594
|
+
SELECT r.flow_slug, st.message_id
|
|
595
|
+
FROM pgflow.step_tasks st
|
|
596
|
+
JOIN pgflow.runs r ON st.run_id = r.run_id
|
|
597
|
+
WHERE st.run_id = complete_task.run_id
|
|
598
|
+
AND st.step_slug = complete_task.step_slug
|
|
599
|
+
AND st.task_index = complete_task.task_index
|
|
600
|
+
AND st.status = 'completed'
|
|
601
|
+
)
|
|
602
|
+
SELECT pgmq.archive(ct.flow_slug, ct.message_id)
|
|
603
|
+
FROM completed_tasks ct
|
|
604
|
+
WHERE EXISTS (SELECT 1 FROM completed_tasks)
|
|
605
|
+
);
|
|
606
|
+
|
|
607
|
+
-- ---------- Trigger next steps ----------
|
|
608
|
+
-- Start any steps that are now ready (deps satisfied)
|
|
609
|
+
PERFORM pgflow.start_ready_steps(complete_task.run_id);
|
|
610
|
+
|
|
611
|
+
-- Check if the entire run is complete
|
|
612
|
+
PERFORM pgflow.maybe_complete_run(complete_task.run_id);
|
|
613
|
+
|
|
614
|
+
-- ---------- Return completed task ----------
|
|
615
|
+
RETURN QUERY SELECT *
|
|
616
|
+
FROM pgflow.step_tasks AS step_task
|
|
617
|
+
WHERE step_task.run_id = complete_task.run_id
|
|
618
|
+
AND step_task.step_slug = complete_task.step_slug
|
|
619
|
+
AND step_task.task_index = complete_task.task_index;
|
|
620
|
+
|
|
621
|
+
end;
|
|
622
|
+
$$;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pgflow/core",
|
|
3
|
-
"version": "0.7.
|
|
4
|
-
"license": "
|
|
3
|
+
"version": "0.7.3",
|
|
4
|
+
"license": "Apache-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"module": "./dist/index.js",
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"postgres": "^3.4.5",
|
|
27
|
-
"@pgflow/dsl": "0.7.
|
|
27
|
+
"@pgflow/dsl": "0.7.3"
|
|
28
28
|
},
|
|
29
29
|
"publishConfig": {
|
|
30
30
|
"access": "public"
|