@pgflow/core 0.0.0-array-map-steps-302d00a8-20250925065142 → 0.0.0-array-map-steps-b956f8f9-20251006084236
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -72
- package/dist/CHANGELOG.md +17 -15
- package/dist/README.md +148 -72
- package/dist/package.json +1 -1
- package/dist/supabase/migrations/{20250919101802_pgflow_temp_orphaned_messages_index.sql → 20251006073122_pgflow_add_map_step_type.sql} +533 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
- package/dist/PLAN_race_condition_testing.md +0 -176
- package/dist/supabase/migrations/20250912075001_pgflow_temp_pr1_schema.sql +0 -185
- package/dist/supabase/migrations/20250912080800_pgflow_temp_pr2_root_maps.sql +0 -95
- package/dist/supabase/migrations/20250912125339_pgflow_TEMP_task_spawning_optimization.sql +0 -146
- package/dist/supabase/migrations/20250916093518_pgflow_temp_add_cascade_complete.sql +0 -321
- package/dist/supabase/migrations/20250916142327_pgflow_temp_make_initial_tasks_nullable.sql +0 -624
- package/dist/supabase/migrations/20250916203905_pgflow_temp_handle_arrays_in_start_tasks.sql +0 -157
- package/dist/supabase/migrations/20250918042753_pgflow_temp_handle_map_output_aggregation.sql +0 -489
- package/dist/supabase/migrations/20250919135211_pgflow_temp_return_task_index_in_start_tasks.sql +0 -178
|
@@ -1,624 +0,0 @@
|
|
|
1
|
-
-- Modify "step_states" table
|
|
2
|
-
ALTER TABLE "pgflow"."step_states" DROP CONSTRAINT "step_states_initial_tasks_check", ADD CONSTRAINT "step_states_initial_tasks_check" CHECK ((initial_tasks IS NULL) OR (initial_tasks >= 0)), ADD CONSTRAINT "initial_tasks_known_when_started" CHECK ((status <> 'started'::text) OR (initial_tasks IS NOT NULL)), ALTER COLUMN "initial_tasks" DROP NOT NULL, ALTER COLUMN "initial_tasks" DROP DEFAULT;
|
|
3
|
-
-- Modify "cascade_complete_taskless_steps" function
|
|
4
|
-
CREATE OR REPLACE FUNCTION "pgflow"."cascade_complete_taskless_steps" ("run_id" uuid) RETURNS integer LANGUAGE plpgsql AS $$
|
|
5
|
-
DECLARE
|
|
6
|
-
v_total_completed int := 0;
|
|
7
|
-
v_iteration_completed int;
|
|
8
|
-
v_iterations int := 0;
|
|
9
|
-
v_max_iterations int := 50;
|
|
10
|
-
BEGIN
|
|
11
|
-
-- ==========================================
|
|
12
|
-
-- ITERATIVE CASCADE COMPLETION
|
|
13
|
-
-- ==========================================
|
|
14
|
-
-- Completes taskless steps in waves until none remain
|
|
15
|
-
LOOP
|
|
16
|
-
-- ---------- Safety check ----------
|
|
17
|
-
v_iterations := v_iterations + 1;
|
|
18
|
-
IF v_iterations > v_max_iterations THEN
|
|
19
|
-
RAISE EXCEPTION 'Cascade loop exceeded safety limit of % iterations', v_max_iterations;
|
|
20
|
-
END IF;
|
|
21
|
-
|
|
22
|
-
-- ==========================================
|
|
23
|
-
-- COMPLETE READY TASKLESS STEPS
|
|
24
|
-
-- ==========================================
|
|
25
|
-
WITH completed AS (
|
|
26
|
-
-- ---------- Complete taskless steps ----------
|
|
27
|
-
-- Steps with initial_tasks=0 and no remaining deps
|
|
28
|
-
UPDATE pgflow.step_states ss
|
|
29
|
-
SET status = 'completed',
|
|
30
|
-
started_at = now(),
|
|
31
|
-
completed_at = now(),
|
|
32
|
-
remaining_tasks = 0
|
|
33
|
-
FROM pgflow.steps s
|
|
34
|
-
WHERE ss.run_id = cascade_complete_taskless_steps.run_id
|
|
35
|
-
AND ss.flow_slug = s.flow_slug
|
|
36
|
-
AND ss.step_slug = s.step_slug
|
|
37
|
-
AND ss.status = 'created'
|
|
38
|
-
AND ss.remaining_deps = 0
|
|
39
|
-
AND ss.initial_tasks = 0
|
|
40
|
-
-- Process in topological order to ensure proper cascade
|
|
41
|
-
RETURNING ss.*
|
|
42
|
-
),
|
|
43
|
-
-- ---------- Update dependent steps ----------
|
|
44
|
-
-- Propagate completion and empty arrays to dependents
|
|
45
|
-
dep_updates AS (
|
|
46
|
-
UPDATE pgflow.step_states ss
|
|
47
|
-
SET remaining_deps = ss.remaining_deps - dep_count.count,
|
|
48
|
-
-- If the dependent is a map step and its dependency completed with 0 tasks,
|
|
49
|
-
-- set its initial_tasks to 0 as well
|
|
50
|
-
initial_tasks = CASE
|
|
51
|
-
WHEN s.step_type = 'map' AND dep_count.has_zero_tasks
|
|
52
|
-
THEN 0 -- Empty array propagation
|
|
53
|
-
ELSE ss.initial_tasks -- Keep existing value (including NULL)
|
|
54
|
-
END
|
|
55
|
-
FROM (
|
|
56
|
-
-- Aggregate dependency updates per dependent step
|
|
57
|
-
SELECT
|
|
58
|
-
d.flow_slug,
|
|
59
|
-
d.step_slug as dependent_slug,
|
|
60
|
-
COUNT(*) as count,
|
|
61
|
-
BOOL_OR(c.initial_tasks = 0) as has_zero_tasks
|
|
62
|
-
FROM completed c
|
|
63
|
-
JOIN pgflow.deps d ON d.flow_slug = c.flow_slug
|
|
64
|
-
AND d.dep_slug = c.step_slug
|
|
65
|
-
GROUP BY d.flow_slug, d.step_slug
|
|
66
|
-
) dep_count,
|
|
67
|
-
pgflow.steps s
|
|
68
|
-
WHERE ss.run_id = cascade_complete_taskless_steps.run_id
|
|
69
|
-
AND ss.flow_slug = dep_count.flow_slug
|
|
70
|
-
AND ss.step_slug = dep_count.dependent_slug
|
|
71
|
-
AND s.flow_slug = ss.flow_slug
|
|
72
|
-
AND s.step_slug = ss.step_slug
|
|
73
|
-
),
|
|
74
|
-
-- ---------- Update run counters ----------
|
|
75
|
-
run_updates AS (
|
|
76
|
-
UPDATE pgflow.runs r
|
|
77
|
-
SET remaining_steps = r.remaining_steps - c.completed_count,
|
|
78
|
-
status = CASE
|
|
79
|
-
WHEN r.remaining_steps - c.completed_count = 0
|
|
80
|
-
THEN 'completed'
|
|
81
|
-
ELSE r.status
|
|
82
|
-
END,
|
|
83
|
-
completed_at = CASE
|
|
84
|
-
WHEN r.remaining_steps - c.completed_count = 0
|
|
85
|
-
THEN now()
|
|
86
|
-
ELSE r.completed_at
|
|
87
|
-
END
|
|
88
|
-
FROM (SELECT COUNT(*) AS completed_count FROM completed) c
|
|
89
|
-
WHERE r.run_id = cascade_complete_taskless_steps.run_id
|
|
90
|
-
AND c.completed_count > 0
|
|
91
|
-
)
|
|
92
|
-
-- ---------- Check iteration results ----------
|
|
93
|
-
SELECT COUNT(*) INTO v_iteration_completed FROM completed;
|
|
94
|
-
|
|
95
|
-
EXIT WHEN v_iteration_completed = 0; -- No more steps to complete
|
|
96
|
-
v_total_completed := v_total_completed + v_iteration_completed;
|
|
97
|
-
END LOOP;
|
|
98
|
-
|
|
99
|
-
RETURN v_total_completed;
|
|
100
|
-
END;
|
|
101
|
-
$$;
|
|
102
|
-
-- Modify "maybe_complete_run" function
|
|
103
|
-
CREATE OR REPLACE FUNCTION "pgflow"."maybe_complete_run" ("run_id" uuid) RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
104
|
-
declare
|
|
105
|
-
v_completed_run pgflow.runs%ROWTYPE;
|
|
106
|
-
begin
|
|
107
|
-
-- ==========================================
|
|
108
|
-
-- CHECK AND COMPLETE RUN IF FINISHED
|
|
109
|
-
-- ==========================================
|
|
110
|
-
WITH run_output AS (
|
|
111
|
-
-- ---------- Gather outputs from leaf steps ----------
|
|
112
|
-
-- Leaf steps = steps with no dependents
|
|
113
|
-
SELECT jsonb_object_agg(st.step_slug, st.output) as final_output
|
|
114
|
-
FROM pgflow.step_tasks st
|
|
115
|
-
JOIN pgflow.step_states ss ON ss.run_id = st.run_id AND ss.step_slug = st.step_slug
|
|
116
|
-
JOIN pgflow.runs r ON r.run_id = ss.run_id AND r.flow_slug = ss.flow_slug
|
|
117
|
-
WHERE st.run_id = maybe_complete_run.run_id
|
|
118
|
-
AND st.status = 'completed'
|
|
119
|
-
AND NOT EXISTS (
|
|
120
|
-
SELECT 1
|
|
121
|
-
FROM pgflow.deps d
|
|
122
|
-
WHERE d.flow_slug = ss.flow_slug
|
|
123
|
-
AND d.dep_slug = ss.step_slug
|
|
124
|
-
)
|
|
125
|
-
)
|
|
126
|
-
-- ---------- Complete run if all steps done ----------
|
|
127
|
-
UPDATE pgflow.runs
|
|
128
|
-
SET
|
|
129
|
-
status = 'completed',
|
|
130
|
-
completed_at = now(),
|
|
131
|
-
output = (SELECT final_output FROM run_output)
|
|
132
|
-
WHERE pgflow.runs.run_id = maybe_complete_run.run_id
|
|
133
|
-
AND pgflow.runs.remaining_steps = 0
|
|
134
|
-
AND pgflow.runs.status != 'completed'
|
|
135
|
-
RETURNING * INTO v_completed_run;
|
|
136
|
-
|
|
137
|
-
-- ==========================================
|
|
138
|
-
-- BROADCAST COMPLETION EVENT
|
|
139
|
-
-- ==========================================
|
|
140
|
-
IF v_completed_run.run_id IS NOT NULL THEN
|
|
141
|
-
PERFORM realtime.send(
|
|
142
|
-
jsonb_build_object(
|
|
143
|
-
'event_type', 'run:completed',
|
|
144
|
-
'run_id', v_completed_run.run_id,
|
|
145
|
-
'flow_slug', v_completed_run.flow_slug,
|
|
146
|
-
'status', 'completed',
|
|
147
|
-
'output', v_completed_run.output,
|
|
148
|
-
'completed_at', v_completed_run.completed_at
|
|
149
|
-
),
|
|
150
|
-
'run:completed',
|
|
151
|
-
concat('pgflow:run:', v_completed_run.run_id),
|
|
152
|
-
false
|
|
153
|
-
);
|
|
154
|
-
END IF;
|
|
155
|
-
end;
|
|
156
|
-
$$;
|
|
157
|
-
-- Modify "start_ready_steps" function
|
|
158
|
-
CREATE OR REPLACE FUNCTION "pgflow"."start_ready_steps" ("run_id" uuid) RETURNS void LANGUAGE sql SET "search_path" = '' AS $$
|
|
159
|
-
-- ==========================================
|
|
160
|
-
-- HANDLE EMPTY ARRAY MAPS (initial_tasks = 0)
|
|
161
|
-
-- ==========================================
|
|
162
|
-
-- These complete immediately without spawning tasks
|
|
163
|
-
WITH empty_map_steps AS (
|
|
164
|
-
SELECT step_state.*
|
|
165
|
-
FROM pgflow.step_states AS step_state
|
|
166
|
-
JOIN pgflow.steps AS step
|
|
167
|
-
ON step.flow_slug = step_state.flow_slug
|
|
168
|
-
AND step.step_slug = step_state.step_slug
|
|
169
|
-
WHERE step_state.run_id = start_ready_steps.run_id
|
|
170
|
-
AND step_state.status = 'created'
|
|
171
|
-
AND step_state.remaining_deps = 0
|
|
172
|
-
AND step.step_type = 'map'
|
|
173
|
-
AND step_state.initial_tasks = 0
|
|
174
|
-
ORDER BY step_state.step_slug
|
|
175
|
-
FOR UPDATE OF step_state
|
|
176
|
-
),
|
|
177
|
-
-- ---------- Complete empty map steps ----------
|
|
178
|
-
completed_empty_steps AS (
|
|
179
|
-
UPDATE pgflow.step_states
|
|
180
|
-
SET status = 'completed',
|
|
181
|
-
started_at = now(),
|
|
182
|
-
completed_at = now(),
|
|
183
|
-
remaining_tasks = 0
|
|
184
|
-
FROM empty_map_steps
|
|
185
|
-
WHERE pgflow.step_states.run_id = start_ready_steps.run_id
|
|
186
|
-
AND pgflow.step_states.step_slug = empty_map_steps.step_slug
|
|
187
|
-
RETURNING pgflow.step_states.*
|
|
188
|
-
),
|
|
189
|
-
-- ---------- Broadcast completion events ----------
|
|
190
|
-
broadcast_empty_completed AS (
|
|
191
|
-
SELECT
|
|
192
|
-
realtime.send(
|
|
193
|
-
jsonb_build_object(
|
|
194
|
-
'event_type', 'step:completed',
|
|
195
|
-
'run_id', completed_step.run_id,
|
|
196
|
-
'step_slug', completed_step.step_slug,
|
|
197
|
-
'status', 'completed',
|
|
198
|
-
'started_at', completed_step.started_at,
|
|
199
|
-
'completed_at', completed_step.completed_at,
|
|
200
|
-
'remaining_tasks', 0,
|
|
201
|
-
'remaining_deps', 0,
|
|
202
|
-
'output', '[]'::jsonb
|
|
203
|
-
),
|
|
204
|
-
concat('step:', completed_step.step_slug, ':completed'),
|
|
205
|
-
concat('pgflow:run:', completed_step.run_id),
|
|
206
|
-
false
|
|
207
|
-
)
|
|
208
|
-
FROM completed_empty_steps AS completed_step
|
|
209
|
-
),
|
|
210
|
-
|
|
211
|
-
-- ==========================================
|
|
212
|
-
-- HANDLE NORMAL STEPS (initial_tasks > 0)
|
|
213
|
-
-- ==========================================
|
|
214
|
-
-- ---------- Find ready steps ----------
|
|
215
|
-
-- Steps with no remaining deps and known task count
|
|
216
|
-
ready_steps AS (
|
|
217
|
-
SELECT *
|
|
218
|
-
FROM pgflow.step_states AS step_state
|
|
219
|
-
WHERE step_state.run_id = start_ready_steps.run_id
|
|
220
|
-
AND step_state.status = 'created'
|
|
221
|
-
AND step_state.remaining_deps = 0
|
|
222
|
-
AND step_state.initial_tasks IS NOT NULL -- NEW: Cannot start with unknown count
|
|
223
|
-
AND step_state.initial_tasks > 0 -- Don't start taskless steps
|
|
224
|
-
-- Exclude empty map steps already handled
|
|
225
|
-
AND NOT EXISTS (
|
|
226
|
-
SELECT 1 FROM empty_map_steps
|
|
227
|
-
WHERE empty_map_steps.run_id = step_state.run_id
|
|
228
|
-
AND empty_map_steps.step_slug = step_state.step_slug
|
|
229
|
-
)
|
|
230
|
-
ORDER BY step_state.step_slug
|
|
231
|
-
FOR UPDATE
|
|
232
|
-
),
|
|
233
|
-
-- ---------- Mark steps as started ----------
|
|
234
|
-
started_step_states AS (
|
|
235
|
-
UPDATE pgflow.step_states
|
|
236
|
-
SET status = 'started',
|
|
237
|
-
started_at = now(),
|
|
238
|
-
remaining_tasks = ready_steps.initial_tasks -- Copy initial_tasks to remaining_tasks when starting
|
|
239
|
-
FROM ready_steps
|
|
240
|
-
WHERE pgflow.step_states.run_id = start_ready_steps.run_id
|
|
241
|
-
AND pgflow.step_states.step_slug = ready_steps.step_slug
|
|
242
|
-
RETURNING pgflow.step_states.*
|
|
243
|
-
),
|
|
244
|
-
|
|
245
|
-
-- ==========================================
|
|
246
|
-
-- TASK GENERATION AND QUEUE MESSAGES
|
|
247
|
-
-- ==========================================
|
|
248
|
-
-- ---------- Generate tasks and batch messages ----------
|
|
249
|
-
-- Single steps: 1 task (index 0)
|
|
250
|
-
-- Map steps: N tasks (indices 0..N-1)
|
|
251
|
-
message_batches AS (
|
|
252
|
-
SELECT
|
|
253
|
-
started_step.flow_slug,
|
|
254
|
-
started_step.run_id,
|
|
255
|
-
started_step.step_slug,
|
|
256
|
-
COALESCE(step.opt_start_delay, 0) as delay,
|
|
257
|
-
array_agg(
|
|
258
|
-
jsonb_build_object(
|
|
259
|
-
'flow_slug', started_step.flow_slug,
|
|
260
|
-
'run_id', started_step.run_id,
|
|
261
|
-
'step_slug', started_step.step_slug,
|
|
262
|
-
'task_index', task_idx.task_index
|
|
263
|
-
) ORDER BY task_idx.task_index
|
|
264
|
-
) AS messages,
|
|
265
|
-
array_agg(task_idx.task_index ORDER BY task_idx.task_index) AS task_indices
|
|
266
|
-
FROM started_step_states AS started_step
|
|
267
|
-
JOIN pgflow.steps AS step
|
|
268
|
-
ON step.flow_slug = started_step.flow_slug
|
|
269
|
-
AND step.step_slug = started_step.step_slug
|
|
270
|
-
-- Generate task indices from 0 to initial_tasks-1
|
|
271
|
-
CROSS JOIN LATERAL generate_series(0, started_step.initial_tasks - 1) AS task_idx(task_index)
|
|
272
|
-
GROUP BY started_step.flow_slug, started_step.run_id, started_step.step_slug, step.opt_start_delay
|
|
273
|
-
),
|
|
274
|
-
-- ---------- Send messages to queue ----------
|
|
275
|
-
-- Uses batch sending for performance with large arrays
|
|
276
|
-
sent_messages AS (
|
|
277
|
-
SELECT
|
|
278
|
-
mb.flow_slug,
|
|
279
|
-
mb.run_id,
|
|
280
|
-
mb.step_slug,
|
|
281
|
-
task_indices.task_index,
|
|
282
|
-
msg_ids.msg_id
|
|
283
|
-
FROM message_batches mb
|
|
284
|
-
CROSS JOIN LATERAL unnest(mb.task_indices) WITH ORDINALITY AS task_indices(task_index, idx_ord)
|
|
285
|
-
CROSS JOIN LATERAL pgmq.send_batch(mb.flow_slug, mb.messages, mb.delay) WITH ORDINALITY AS msg_ids(msg_id, msg_ord)
|
|
286
|
-
WHERE task_indices.idx_ord = msg_ids.msg_ord
|
|
287
|
-
),
|
|
288
|
-
|
|
289
|
-
-- ---------- Broadcast step:started events ----------
|
|
290
|
-
broadcast_events AS (
|
|
291
|
-
SELECT
|
|
292
|
-
realtime.send(
|
|
293
|
-
jsonb_build_object(
|
|
294
|
-
'event_type', 'step:started',
|
|
295
|
-
'run_id', started_step.run_id,
|
|
296
|
-
'step_slug', started_step.step_slug,
|
|
297
|
-
'status', 'started',
|
|
298
|
-
'started_at', started_step.started_at,
|
|
299
|
-
'remaining_tasks', started_step.remaining_tasks,
|
|
300
|
-
'remaining_deps', started_step.remaining_deps
|
|
301
|
-
),
|
|
302
|
-
concat('step:', started_step.step_slug, ':started'),
|
|
303
|
-
concat('pgflow:run:', started_step.run_id),
|
|
304
|
-
false
|
|
305
|
-
)
|
|
306
|
-
FROM started_step_states AS started_step
|
|
307
|
-
)
|
|
308
|
-
|
|
309
|
-
-- ==========================================
|
|
310
|
-
-- RECORD TASKS IN DATABASE
|
|
311
|
-
-- ==========================================
|
|
312
|
-
INSERT INTO pgflow.step_tasks (flow_slug, run_id, step_slug, task_index, message_id)
|
|
313
|
-
SELECT
|
|
314
|
-
sent_messages.flow_slug,
|
|
315
|
-
sent_messages.run_id,
|
|
316
|
-
sent_messages.step_slug,
|
|
317
|
-
sent_messages.task_index,
|
|
318
|
-
sent_messages.msg_id
|
|
319
|
-
FROM sent_messages;
|
|
320
|
-
$$;
|
|
321
|
-
-- Modify "complete_task" function
|
|
322
|
-
CREATE OR REPLACE FUNCTION "pgflow"."complete_task" ("run_id" uuid, "step_slug" text, "task_index" integer, "output" jsonb) RETURNS SETOF "pgflow"."step_tasks" LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
323
|
-
declare
|
|
324
|
-
v_step_state pgflow.step_states%ROWTYPE;
|
|
325
|
-
v_dependent_map_slug text;
|
|
326
|
-
begin
|
|
327
|
-
|
|
328
|
-
-- ==========================================
|
|
329
|
-
-- VALIDATION: Array output for dependent maps
|
|
330
|
-
-- ==========================================
|
|
331
|
-
-- Must happen BEFORE acquiring locks to fail fast without holding resources
|
|
332
|
-
SELECT ds.step_slug INTO v_dependent_map_slug
|
|
333
|
-
FROM pgflow.deps d
|
|
334
|
-
JOIN pgflow.steps ds ON ds.flow_slug = d.flow_slug AND ds.step_slug = d.step_slug
|
|
335
|
-
JOIN pgflow.step_states ss ON ss.flow_slug = ds.flow_slug AND ss.step_slug = ds.step_slug
|
|
336
|
-
WHERE d.dep_slug = complete_task.step_slug
|
|
337
|
-
AND d.flow_slug = (SELECT r.flow_slug FROM pgflow.runs r WHERE r.run_id = complete_task.run_id)
|
|
338
|
-
AND ds.step_type = 'map'
|
|
339
|
-
AND ss.run_id = complete_task.run_id
|
|
340
|
-
AND ss.initial_tasks IS NULL
|
|
341
|
-
AND (complete_task.output IS NULL OR jsonb_typeof(complete_task.output) != 'array')
|
|
342
|
-
LIMIT 1;
|
|
343
|
-
|
|
344
|
-
IF v_dependent_map_slug IS NOT NULL THEN
|
|
345
|
-
RAISE EXCEPTION 'Map step % expects array input but dependency % produced % (output: %)',
|
|
346
|
-
v_dependent_map_slug,
|
|
347
|
-
complete_task.step_slug,
|
|
348
|
-
CASE WHEN complete_task.output IS NULL THEN 'null' ELSE jsonb_typeof(complete_task.output) END,
|
|
349
|
-
complete_task.output;
|
|
350
|
-
END IF;
|
|
351
|
-
|
|
352
|
-
-- ==========================================
|
|
353
|
-
-- MAIN CTE CHAIN: Update task and propagate changes
|
|
354
|
-
-- ==========================================
|
|
355
|
-
WITH
|
|
356
|
-
-- ---------- Lock acquisition ----------
|
|
357
|
-
-- Acquire locks in consistent order (run -> step) to prevent deadlocks
|
|
358
|
-
run_lock AS (
|
|
359
|
-
SELECT * FROM pgflow.runs
|
|
360
|
-
WHERE pgflow.runs.run_id = complete_task.run_id
|
|
361
|
-
FOR UPDATE
|
|
362
|
-
),
|
|
363
|
-
step_lock AS (
|
|
364
|
-
SELECT * FROM pgflow.step_states
|
|
365
|
-
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
366
|
-
AND pgflow.step_states.step_slug = complete_task.step_slug
|
|
367
|
-
FOR UPDATE
|
|
368
|
-
),
|
|
369
|
-
-- ---------- Task completion ----------
|
|
370
|
-
-- Update the task record with completion status and output
|
|
371
|
-
task AS (
|
|
372
|
-
UPDATE pgflow.step_tasks
|
|
373
|
-
SET
|
|
374
|
-
status = 'completed',
|
|
375
|
-
completed_at = now(),
|
|
376
|
-
output = complete_task.output
|
|
377
|
-
WHERE pgflow.step_tasks.run_id = complete_task.run_id
|
|
378
|
-
AND pgflow.step_tasks.step_slug = complete_task.step_slug
|
|
379
|
-
AND pgflow.step_tasks.task_index = complete_task.task_index
|
|
380
|
-
AND pgflow.step_tasks.status = 'started'
|
|
381
|
-
RETURNING *
|
|
382
|
-
),
|
|
383
|
-
-- ---------- Step state update ----------
|
|
384
|
-
-- Decrement remaining_tasks and potentially mark step as completed
|
|
385
|
-
step_state AS (
|
|
386
|
-
UPDATE pgflow.step_states
|
|
387
|
-
SET
|
|
388
|
-
status = CASE
|
|
389
|
-
WHEN pgflow.step_states.remaining_tasks = 1 THEN 'completed' -- Will be 0 after decrement
|
|
390
|
-
ELSE 'started'
|
|
391
|
-
END,
|
|
392
|
-
completed_at = CASE
|
|
393
|
-
WHEN pgflow.step_states.remaining_tasks = 1 THEN now() -- Will be 0 after decrement
|
|
394
|
-
ELSE NULL
|
|
395
|
-
END,
|
|
396
|
-
remaining_tasks = pgflow.step_states.remaining_tasks - 1
|
|
397
|
-
FROM task
|
|
398
|
-
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
399
|
-
AND pgflow.step_states.step_slug = complete_task.step_slug
|
|
400
|
-
RETURNING pgflow.step_states.*
|
|
401
|
-
),
|
|
402
|
-
-- ---------- Dependency resolution ----------
|
|
403
|
-
-- Find all steps that depend on the completed step (only if step completed)
|
|
404
|
-
dependent_steps AS (
|
|
405
|
-
SELECT d.step_slug AS dependent_step_slug
|
|
406
|
-
FROM pgflow.deps d
|
|
407
|
-
JOIN step_state s ON s.status = 'completed' AND d.flow_slug = s.flow_slug
|
|
408
|
-
WHERE d.dep_slug = complete_task.step_slug
|
|
409
|
-
ORDER BY d.step_slug -- Ensure consistent ordering
|
|
410
|
-
),
|
|
411
|
-
-- ---------- Lock dependent steps ----------
|
|
412
|
-
-- Acquire locks on all dependent steps before updating them
|
|
413
|
-
dependent_steps_lock AS (
|
|
414
|
-
SELECT * FROM pgflow.step_states
|
|
415
|
-
WHERE pgflow.step_states.run_id = complete_task.run_id
|
|
416
|
-
AND pgflow.step_states.step_slug IN (SELECT dependent_step_slug FROM dependent_steps)
|
|
417
|
-
FOR UPDATE
|
|
418
|
-
),
|
|
419
|
-
-- ---------- Update dependent steps ----------
|
|
420
|
-
-- Decrement remaining_deps and resolve NULL initial_tasks for map steps
|
|
421
|
-
dependent_steps_update AS (
|
|
422
|
-
UPDATE pgflow.step_states ss
|
|
423
|
-
SET remaining_deps = ss.remaining_deps - 1,
|
|
424
|
-
-- Resolve NULL initial_tasks for dependent map steps
|
|
425
|
-
-- This is where dependent maps learn their array size from upstream
|
|
426
|
-
initial_tasks = CASE
|
|
427
|
-
WHEN s.step_type = 'map' AND ss.initial_tasks IS NULL
|
|
428
|
-
AND complete_task.output IS NOT NULL
|
|
429
|
-
AND jsonb_typeof(complete_task.output) = 'array' THEN
|
|
430
|
-
jsonb_array_length(complete_task.output)
|
|
431
|
-
ELSE ss.initial_tasks -- Keep existing value (including NULL)
|
|
432
|
-
END
|
|
433
|
-
FROM dependent_steps ds, pgflow.steps s
|
|
434
|
-
WHERE ss.run_id = complete_task.run_id
|
|
435
|
-
AND ss.step_slug = ds.dependent_step_slug
|
|
436
|
-
AND s.flow_slug = ss.flow_slug
|
|
437
|
-
AND s.step_slug = ss.step_slug
|
|
438
|
-
)
|
|
439
|
-
-- ---------- Update run remaining_steps ----------
|
|
440
|
-
-- Decrement the run's remaining_steps counter if step completed
|
|
441
|
-
UPDATE pgflow.runs
|
|
442
|
-
SET remaining_steps = pgflow.runs.remaining_steps - 1
|
|
443
|
-
FROM step_state
|
|
444
|
-
WHERE pgflow.runs.run_id = complete_task.run_id
|
|
445
|
-
AND step_state.status = 'completed';
|
|
446
|
-
|
|
447
|
-
-- ==========================================
|
|
448
|
-
-- POST-COMPLETION ACTIONS
|
|
449
|
-
-- ==========================================
|
|
450
|
-
|
|
451
|
-
-- ---------- Get updated state for broadcasting ----------
|
|
452
|
-
SELECT * INTO v_step_state FROM pgflow.step_states
|
|
453
|
-
WHERE pgflow.step_states.run_id = complete_task.run_id AND pgflow.step_states.step_slug = complete_task.step_slug;
|
|
454
|
-
|
|
455
|
-
-- ---------- Handle step completion ----------
|
|
456
|
-
IF v_step_state.status = 'completed' THEN
|
|
457
|
-
-- Cascade complete any taskless steps that are now ready
|
|
458
|
-
PERFORM pgflow.cascade_complete_taskless_steps(complete_task.run_id);
|
|
459
|
-
|
|
460
|
-
-- Broadcast step:completed event
|
|
461
|
-
PERFORM realtime.send(
|
|
462
|
-
jsonb_build_object(
|
|
463
|
-
'event_type', 'step:completed',
|
|
464
|
-
'run_id', complete_task.run_id,
|
|
465
|
-
'step_slug', complete_task.step_slug,
|
|
466
|
-
'status', 'completed',
|
|
467
|
-
'output', complete_task.output,
|
|
468
|
-
'completed_at', v_step_state.completed_at
|
|
469
|
-
),
|
|
470
|
-
concat('step:', complete_task.step_slug, ':completed'),
|
|
471
|
-
concat('pgflow:run:', complete_task.run_id),
|
|
472
|
-
false
|
|
473
|
-
);
|
|
474
|
-
END IF;
|
|
475
|
-
|
|
476
|
-
-- ---------- Archive completed task message ----------
|
|
477
|
-
-- Move message from active queue to archive table
|
|
478
|
-
PERFORM (
|
|
479
|
-
WITH completed_tasks AS (
|
|
480
|
-
SELECT r.flow_slug, st.message_id
|
|
481
|
-
FROM pgflow.step_tasks st
|
|
482
|
-
JOIN pgflow.runs r ON st.run_id = r.run_id
|
|
483
|
-
WHERE st.run_id = complete_task.run_id
|
|
484
|
-
AND st.step_slug = complete_task.step_slug
|
|
485
|
-
AND st.task_index = complete_task.task_index
|
|
486
|
-
AND st.status = 'completed'
|
|
487
|
-
)
|
|
488
|
-
SELECT pgmq.archive(ct.flow_slug, ct.message_id)
|
|
489
|
-
FROM completed_tasks ct
|
|
490
|
-
WHERE EXISTS (SELECT 1 FROM completed_tasks)
|
|
491
|
-
);
|
|
492
|
-
|
|
493
|
-
-- ---------- Trigger next steps ----------
|
|
494
|
-
-- Start any steps that are now ready (deps satisfied)
|
|
495
|
-
PERFORM pgflow.start_ready_steps(complete_task.run_id);
|
|
496
|
-
|
|
497
|
-
-- Check if the entire run is complete
|
|
498
|
-
PERFORM pgflow.maybe_complete_run(complete_task.run_id);
|
|
499
|
-
|
|
500
|
-
-- ---------- Return completed task ----------
|
|
501
|
-
RETURN QUERY SELECT *
|
|
502
|
-
FROM pgflow.step_tasks AS step_task
|
|
503
|
-
WHERE step_task.run_id = complete_task.run_id
|
|
504
|
-
AND step_task.step_slug = complete_task.step_slug
|
|
505
|
-
AND step_task.task_index = complete_task.task_index;
|
|
506
|
-
|
|
507
|
-
end;
|
|
508
|
-
$$;
|
|
509
|
-
-- Modify "start_flow" function
|
|
510
|
-
CREATE OR REPLACE FUNCTION "pgflow"."start_flow" ("flow_slug" text, "input" jsonb, "run_id" uuid DEFAULT NULL::uuid) RETURNS SETOF "pgflow"."runs" LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
511
|
-
declare
|
|
512
|
-
v_created_run pgflow.runs%ROWTYPE;
|
|
513
|
-
v_root_map_count int;
|
|
514
|
-
begin
|
|
515
|
-
|
|
516
|
-
-- ==========================================
|
|
517
|
-
-- VALIDATION: Root map array input
|
|
518
|
-
-- ==========================================
|
|
519
|
-
WITH root_maps AS (
|
|
520
|
-
SELECT step_slug
|
|
521
|
-
FROM pgflow.steps
|
|
522
|
-
WHERE steps.flow_slug = start_flow.flow_slug
|
|
523
|
-
AND steps.step_type = 'map'
|
|
524
|
-
AND steps.deps_count = 0
|
|
525
|
-
)
|
|
526
|
-
SELECT COUNT(*) INTO v_root_map_count FROM root_maps;
|
|
527
|
-
|
|
528
|
-
-- If we have root map steps, validate that input is an array
|
|
529
|
-
IF v_root_map_count > 0 THEN
|
|
530
|
-
-- First check for NULL (should be caught by NOT NULL constraint, but be defensive)
|
|
531
|
-
IF start_flow.input IS NULL THEN
|
|
532
|
-
RAISE EXCEPTION 'Flow % has root map steps but input is NULL', start_flow.flow_slug;
|
|
533
|
-
END IF;
|
|
534
|
-
|
|
535
|
-
-- Then check if it's not an array
|
|
536
|
-
IF jsonb_typeof(start_flow.input) != 'array' THEN
|
|
537
|
-
RAISE EXCEPTION 'Flow % has root map steps but input is not an array (got %)',
|
|
538
|
-
start_flow.flow_slug, jsonb_typeof(start_flow.input);
|
|
539
|
-
END IF;
|
|
540
|
-
END IF;
|
|
541
|
-
|
|
542
|
-
-- ==========================================
|
|
543
|
-
-- MAIN CTE CHAIN: Create run and step states
|
|
544
|
-
-- ==========================================
|
|
545
|
-
WITH
|
|
546
|
-
-- ---------- Gather flow metadata ----------
|
|
547
|
-
flow_steps AS (
|
|
548
|
-
SELECT steps.flow_slug, steps.step_slug, steps.step_type, steps.deps_count
|
|
549
|
-
FROM pgflow.steps
|
|
550
|
-
WHERE steps.flow_slug = start_flow.flow_slug
|
|
551
|
-
),
|
|
552
|
-
-- ---------- Create run record ----------
|
|
553
|
-
created_run AS (
|
|
554
|
-
INSERT INTO pgflow.runs (run_id, flow_slug, input, remaining_steps)
|
|
555
|
-
VALUES (
|
|
556
|
-
COALESCE(start_flow.run_id, gen_random_uuid()),
|
|
557
|
-
start_flow.flow_slug,
|
|
558
|
-
start_flow.input,
|
|
559
|
-
(SELECT count(*) FROM flow_steps)
|
|
560
|
-
)
|
|
561
|
-
RETURNING *
|
|
562
|
-
),
|
|
563
|
-
-- ---------- Create step states ----------
|
|
564
|
-
-- Sets initial_tasks: known for root maps, NULL for dependent maps
|
|
565
|
-
created_step_states AS (
|
|
566
|
-
INSERT INTO pgflow.step_states (flow_slug, run_id, step_slug, remaining_deps, initial_tasks)
|
|
567
|
-
SELECT
|
|
568
|
-
fs.flow_slug,
|
|
569
|
-
(SELECT created_run.run_id FROM created_run),
|
|
570
|
-
fs.step_slug,
|
|
571
|
-
fs.deps_count,
|
|
572
|
-
-- Updated logic for initial_tasks:
|
|
573
|
-
CASE
|
|
574
|
-
WHEN fs.step_type = 'map' AND fs.deps_count = 0 THEN
|
|
575
|
-
-- Root map: get array length from input
|
|
576
|
-
CASE
|
|
577
|
-
WHEN jsonb_typeof(start_flow.input) = 'array' THEN
|
|
578
|
-
jsonb_array_length(start_flow.input)
|
|
579
|
-
ELSE
|
|
580
|
-
1
|
|
581
|
-
END
|
|
582
|
-
WHEN fs.step_type = 'map' AND fs.deps_count > 0 THEN
|
|
583
|
-
-- Dependent map: unknown until dependencies complete
|
|
584
|
-
NULL
|
|
585
|
-
ELSE
|
|
586
|
-
-- Single steps: always 1 task
|
|
587
|
-
1
|
|
588
|
-
END
|
|
589
|
-
FROM flow_steps fs
|
|
590
|
-
)
|
|
591
|
-
SELECT * FROM created_run INTO v_created_run;
|
|
592
|
-
|
|
593
|
-
-- ==========================================
|
|
594
|
-
-- POST-CREATION ACTIONS
|
|
595
|
-
-- ==========================================
|
|
596
|
-
|
|
597
|
-
-- ---------- Broadcast run:started event ----------
|
|
598
|
-
PERFORM realtime.send(
|
|
599
|
-
jsonb_build_object(
|
|
600
|
-
'event_type', 'run:started',
|
|
601
|
-
'run_id', v_created_run.run_id,
|
|
602
|
-
'flow_slug', v_created_run.flow_slug,
|
|
603
|
-
'input', v_created_run.input,
|
|
604
|
-
'status', 'started',
|
|
605
|
-
'remaining_steps', v_created_run.remaining_steps,
|
|
606
|
-
'started_at', v_created_run.started_at
|
|
607
|
-
),
|
|
608
|
-
'run:started',
|
|
609
|
-
concat('pgflow:run:', v_created_run.run_id),
|
|
610
|
-
false
|
|
611
|
-
);
|
|
612
|
-
|
|
613
|
-
-- ---------- Complete taskless steps ----------
|
|
614
|
-
-- Handle empty array maps that should auto-complete
|
|
615
|
-
PERFORM pgflow.cascade_complete_taskless_steps(v_created_run.run_id);
|
|
616
|
-
|
|
617
|
-
-- ---------- Start initial steps ----------
|
|
618
|
-
-- Start root steps (those with no dependencies)
|
|
619
|
-
PERFORM pgflow.start_ready_steps(v_created_run.run_id);
|
|
620
|
-
|
|
621
|
-
RETURN QUERY SELECT * FROM pgflow.runs where pgflow.runs.run_id = v_created_run.run_id;
|
|
622
|
-
|
|
623
|
-
end;
|
|
624
|
-
$$;
|