@pgflow/core 0.0.0-pgflow-installer-45a8ec76-20251211182851 → 0.0.0-pgflow-installer-249c293d-20251211201430

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/CHANGELOG.md +2 -2
  2. package/dist/package.json +1 -5
  3. package/package.json +2 -6
  4. package/dist/migrations/_generated/m_20250429164909.d.ts +0 -3
  5. package/dist/migrations/_generated/m_20250429164909.d.ts.map +0 -1
  6. package/dist/migrations/_generated/m_20250429164909.js +0 -584
  7. package/dist/migrations/_generated/m_20250517072017.d.ts +0 -3
  8. package/dist/migrations/_generated/m_20250517072017.d.ts.map +0 -1
  9. package/dist/migrations/_generated/m_20250517072017.js +0 -106
  10. package/dist/migrations/_generated/m_20250609105135.d.ts +0 -3
  11. package/dist/migrations/_generated/m_20250609105135.d.ts.map +0 -1
  12. package/dist/migrations/_generated/m_20250609105135.js +0 -376
  13. package/dist/migrations/_generated/m_20250610180554.d.ts +0 -3
  14. package/dist/migrations/_generated/m_20250610180554.d.ts.map +0 -1
  15. package/dist/migrations/_generated/m_20250610180554.js +0 -132
  16. package/dist/migrations/_generated/m_20250614124241.d.ts +0 -3
  17. package/dist/migrations/_generated/m_20250614124241.d.ts.map +0 -1
  18. package/dist/migrations/_generated/m_20250614124241.js +0 -506
  19. package/dist/migrations/_generated/m_20250619195327.d.ts +0 -3
  20. package/dist/migrations/_generated/m_20250619195327.d.ts.map +0 -1
  21. package/dist/migrations/_generated/m_20250619195327.js +0 -190
  22. package/dist/migrations/_generated/m_20250627090700.d.ts +0 -3
  23. package/dist/migrations/_generated/m_20250627090700.d.ts.map +0 -1
  24. package/dist/migrations/_generated/m_20250627090700.js +0 -11
  25. package/dist/migrations/_generated/m_20250707210212.d.ts +0 -3
  26. package/dist/migrations/_generated/m_20250707210212.d.ts.map +0 -1
  27. package/dist/migrations/_generated/m_20250707210212.js +0 -108
  28. package/dist/migrations/_generated/m_20250719205006.d.ts +0 -3
  29. package/dist/migrations/_generated/m_20250719205006.d.ts.map +0 -1
  30. package/dist/migrations/_generated/m_20250719205006.js +0 -7
  31. package/dist/migrations/_generated/m_20251006073122.d.ts +0 -3
  32. package/dist/migrations/_generated/m_20251006073122.d.ts.map +0 -1
  33. package/dist/migrations/_generated/m_20251006073122.js +0 -1249
  34. package/dist/migrations/_generated/m_20251103222045.d.ts +0 -3
  35. package/dist/migrations/_generated/m_20251103222045.d.ts.map +0 -1
  36. package/dist/migrations/_generated/m_20251103222045.js +0 -627
  37. package/dist/migrations/_generated/m_20251104080523.d.ts +0 -3
  38. package/dist/migrations/_generated/m_20251104080523.d.ts.map +0 -1
  39. package/dist/migrations/_generated/m_20251104080523.js +0 -98
  40. package/dist/migrations/_generated/m_20251130000000.d.ts +0 -3
  41. package/dist/migrations/_generated/m_20251130000000.d.ts.map +0 -1
  42. package/dist/migrations/_generated/m_20251130000000.js +0 -273
  43. package/dist/migrations/_generated/m_20251209074533.d.ts +0 -3
  44. package/dist/migrations/_generated/m_20251209074533.d.ts.map +0 -1
  45. package/dist/migrations/_generated/m_20251209074533.js +0 -278
  46. package/dist/migrations/index.d.ts +0 -11
  47. package/dist/migrations/index.d.ts.map +0 -1
  48. package/dist/migrations/index.js +0 -39
  49. package/dist/migrations/types.d.ts +0 -12
  50. package/dist/migrations/types.d.ts.map +0 -1
  51. package/dist/migrations/types.js +0 -1
@@ -1,1249 +0,0 @@
1
- export const migration = {
2
- timestamp: '20251006073122',
3
- filename: '20251006073122_pgflow_add_map_step_type.sql',
4
- content: `-- Modify "step_task_record" composite type
5
- ALTER TYPE "pgflow"."step_task_record" ADD ATTRIBUTE "task_index" integer;
6
- -- Modify "step_states" table - Step 1: Drop old constraint and NOT NULL
7
- ALTER TABLE "pgflow"."step_states"
8
- DROP CONSTRAINT "step_states_remaining_tasks_check",
9
- ALTER COLUMN "remaining_tasks" DROP NOT NULL,
10
- ALTER COLUMN "remaining_tasks" DROP DEFAULT,
11
- ADD COLUMN "initial_tasks" integer NULL;
12
- -- AUTOMATIC DATA MIGRATION: Prepare existing data for new constraints
13
- -- This runs AFTER dropping NOT NULL but BEFORE adding new constraints
14
- -- All old steps had exactly 1 task (enforced by old only_single_task_per_step constraint)
15
-
16
- -- Backfill initial_tasks = 1 for all existing steps
17
- -- (Old schema enforced exactly 1 task per step, so all steps had initial_tasks=1)
18
- UPDATE "pgflow"."step_states"
19
- SET "initial_tasks" = 1
20
- WHERE "initial_tasks" IS NULL;
21
-
22
- -- Set remaining_tasks to NULL for 'created' status
23
- -- (New semantics: NULL = not started, old semantics: 1 = not started)
24
- UPDATE "pgflow"."step_states"
25
- SET "remaining_tasks" = NULL
26
- WHERE "status" = 'created' AND "remaining_tasks" IS NOT NULL;
27
- -- Modify "step_states" table - Step 2: Add new constraints
28
- ALTER TABLE "pgflow"."step_states"
29
- ADD CONSTRAINT "initial_tasks_known_when_started" CHECK ((status <> 'started'::text) OR (initial_tasks IS NOT NULL)),
30
- ADD CONSTRAINT "remaining_tasks_state_consistency" CHECK ((remaining_tasks IS NULL) OR (status <> 'created'::text)),
31
- ADD CONSTRAINT "step_states_initial_tasks_check" CHECK ((initial_tasks IS NULL) OR (initial_tasks >= 0));
32
- -- Modify "step_tasks" table
33
- ALTER TABLE "pgflow"."step_tasks" DROP CONSTRAINT "only_single_task_per_step", DROP CONSTRAINT "output_valid_only_for_completed", ADD CONSTRAINT "output_valid_only_for_completed" CHECK ((output IS NULL) OR (status = ANY (ARRAY['completed'::text, 'failed'::text])));
34
- -- Modify "steps" table
35
- ALTER TABLE "pgflow"."steps" DROP CONSTRAINT "steps_step_type_check", ADD CONSTRAINT "steps_step_type_check" CHECK (step_type = ANY (ARRAY['single'::text, 'map'::text]));
36
- -- Modify "maybe_complete_run" function
37
- CREATE OR REPLACE FUNCTION "pgflow"."maybe_complete_run" ("run_id" uuid) RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$
38
- declare
39
- v_completed_run pgflow.runs%ROWTYPE;
40
- begin
41
- -- ==========================================
42
- -- CHECK AND COMPLETE RUN IF FINISHED
43
- -- ==========================================
44
- -- ---------- Complete run if all steps done ----------
45
- UPDATE pgflow.runs
46
- SET
47
- status = 'completed',
48
- completed_at = now(),
49
- -- Only compute expensive aggregation when actually completing the run
50
- output = (
51
- -- ---------- Gather outputs from leaf steps ----------
52
- -- Leaf steps = steps with no dependents
53
- -- For map steps: aggregate all task outputs into array
54
- -- For single steps: use the single task output
55
- SELECT jsonb_object_agg(
56
- step_slug,
57
- CASE
58
- WHEN step_type = 'map' THEN aggregated_output
59
- ELSE single_output
60
- END
61
- )
62
- FROM (
63
- SELECT DISTINCT
64
- leaf_state.step_slug,
65
- leaf_step.step_type,
66
- -- For map steps: aggregate all task outputs
67
- CASE WHEN leaf_step.step_type = 'map' THEN
68
- (SELECT COALESCE(jsonb_agg(leaf_task.output ORDER BY leaf_task.task_index), '[]'::jsonb)
69
- FROM pgflow.step_tasks leaf_task
70
- WHERE leaf_task.run_id = leaf_state.run_id
71
- AND leaf_task.step_slug = leaf_state.step_slug
72
- AND leaf_task.status = 'completed')
73
- END as aggregated_output,
74
- -- For single steps: get the single output
75
- CASE WHEN leaf_step.step_type = 'single' THEN
76
- (SELECT leaf_task.output
77
- FROM pgflow.step_tasks leaf_task
78
- WHERE leaf_task.run_id = leaf_state.run_id
79
- AND leaf_task.step_slug = leaf_state.step_slug
80
- AND leaf_task.status = 'completed'
81
- LIMIT 1)
82
- END as single_output
83
- FROM pgflow.step_states leaf_state
84
- JOIN pgflow.steps leaf_step ON leaf_step.flow_slug = leaf_state.flow_slug AND leaf_step.step_slug = leaf_state.step_slug
85
- WHERE leaf_state.run_id = maybe_complete_run.run_id
86
- AND leaf_state.status = 'completed'
87
- AND NOT EXISTS (
88
- SELECT 1
89
- FROM pgflow.deps dep
90
- WHERE dep.flow_slug = leaf_state.flow_slug
91
- AND dep.dep_slug = leaf_state.step_slug
92
- )
93
- ) leaf_outputs
94
- )
95
- WHERE pgflow.runs.run_id = maybe_complete_run.run_id
96
- AND pgflow.runs.remaining_steps = 0
97
- AND pgflow.runs.status != 'completed'
98
- RETURNING * INTO v_completed_run;
99
-
100
- -- ==========================================
101
- -- BROADCAST COMPLETION EVENT
102
- -- ==========================================
103
- IF v_completed_run.run_id IS NOT NULL THEN
104
- PERFORM realtime.send(
105
- jsonb_build_object(
106
- 'event_type', 'run:completed',
107
- 'run_id', v_completed_run.run_id,
108
- 'flow_slug', v_completed_run.flow_slug,
109
- 'status', 'completed',
110
- 'output', v_completed_run.output,
111
- 'completed_at', v_completed_run.completed_at
112
- ),
113
- 'run:completed',
114
- concat('pgflow:run:', v_completed_run.run_id),
115
- false
116
- );
117
- END IF;
118
- end;
119
- $$;
120
- -- Modify "start_ready_steps" function
121
- CREATE OR REPLACE FUNCTION "pgflow"."start_ready_steps" ("run_id" uuid) RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$
122
- begin
123
- -- ==========================================
124
- -- GUARD: No mutations on failed runs
125
- -- ==========================================
126
- IF EXISTS (SELECT 1 FROM pgflow.runs WHERE pgflow.runs.run_id = start_ready_steps.run_id AND pgflow.runs.status = 'failed') THEN
127
- RETURN;
128
- END IF;
129
-
130
- -- ==========================================
131
- -- HANDLE EMPTY ARRAY MAPS (initial_tasks = 0)
132
- -- ==========================================
133
- -- These complete immediately without spawning tasks
134
- WITH empty_map_steps AS (
135
- SELECT step_state.*
136
- FROM pgflow.step_states AS step_state
137
- JOIN pgflow.steps AS step
138
- ON step.flow_slug = step_state.flow_slug
139
- AND step.step_slug = step_state.step_slug
140
- WHERE step_state.run_id = start_ready_steps.run_id
141
- AND step_state.status = 'created'
142
- AND step_state.remaining_deps = 0
143
- AND step.step_type = 'map'
144
- AND step_state.initial_tasks = 0
145
- ORDER BY step_state.step_slug
146
- FOR UPDATE OF step_state
147
- ),
148
- -- ---------- Complete empty map steps ----------
149
- completed_empty_steps AS (
150
- UPDATE pgflow.step_states
151
- SET status = 'completed',
152
- started_at = now(),
153
- completed_at = now(),
154
- remaining_tasks = 0
155
- FROM empty_map_steps
156
- WHERE pgflow.step_states.run_id = start_ready_steps.run_id
157
- AND pgflow.step_states.step_slug = empty_map_steps.step_slug
158
- RETURNING pgflow.step_states.*
159
- ),
160
- -- ---------- Broadcast completion events ----------
161
- broadcast_empty_completed AS (
162
- SELECT
163
- realtime.send(
164
- jsonb_build_object(
165
- 'event_type', 'step:completed',
166
- 'run_id', completed_step.run_id,
167
- 'step_slug', completed_step.step_slug,
168
- 'status', 'completed',
169
- 'started_at', completed_step.started_at,
170
- 'completed_at', completed_step.completed_at,
171
- 'remaining_tasks', 0,
172
- 'remaining_deps', 0,
173
- 'output', '[]'::jsonb
174
- ),
175
- concat('step:', completed_step.step_slug, ':completed'),
176
- concat('pgflow:run:', completed_step.run_id),
177
- false
178
- )
179
- FROM completed_empty_steps AS completed_step
180
- ),
181
-
182
- -- ==========================================
183
- -- HANDLE NORMAL STEPS (initial_tasks > 0)
184
- -- ==========================================
185
- -- ---------- Find ready steps ----------
186
- -- Steps with no remaining deps and known task count
187
- ready_steps AS (
188
- SELECT *
189
- FROM pgflow.step_states AS step_state
190
- WHERE step_state.run_id = start_ready_steps.run_id
191
- AND step_state.status = 'created'
192
- AND step_state.remaining_deps = 0
193
- AND step_state.initial_tasks IS NOT NULL -- NEW: Cannot start with unknown count
194
- AND step_state.initial_tasks > 0 -- Don't start taskless steps
195
- -- Exclude empty map steps already handled
196
- AND NOT EXISTS (
197
- SELECT 1 FROM empty_map_steps
198
- WHERE empty_map_steps.run_id = step_state.run_id
199
- AND empty_map_steps.step_slug = step_state.step_slug
200
- )
201
- ORDER BY step_state.step_slug
202
- FOR UPDATE
203
- ),
204
- -- ---------- Mark steps as started ----------
205
- started_step_states AS (
206
- UPDATE pgflow.step_states
207
- SET status = 'started',
208
- started_at = now(),
209
- remaining_tasks = ready_steps.initial_tasks -- Copy initial_tasks to remaining_tasks when starting
210
- FROM ready_steps
211
- WHERE pgflow.step_states.run_id = start_ready_steps.run_id
212
- AND pgflow.step_states.step_slug = ready_steps.step_slug
213
- RETURNING pgflow.step_states.*
214
- ),
215
-
216
- -- ==========================================
217
- -- TASK GENERATION AND QUEUE MESSAGES
218
- -- ==========================================
219
- -- ---------- Generate tasks and batch messages ----------
220
- -- Single steps: 1 task (index 0)
221
- -- Map steps: N tasks (indices 0..N-1)
222
- message_batches AS (
223
- SELECT
224
- started_step.flow_slug,
225
- started_step.run_id,
226
- started_step.step_slug,
227
- COALESCE(step.opt_start_delay, 0) as delay,
228
- array_agg(
229
- jsonb_build_object(
230
- 'flow_slug', started_step.flow_slug,
231
- 'run_id', started_step.run_id,
232
- 'step_slug', started_step.step_slug,
233
- 'task_index', task_idx.task_index
234
- ) ORDER BY task_idx.task_index
235
- ) AS messages,
236
- array_agg(task_idx.task_index ORDER BY task_idx.task_index) AS task_indices
237
- FROM started_step_states AS started_step
238
- JOIN pgflow.steps AS step
239
- ON step.flow_slug = started_step.flow_slug
240
- AND step.step_slug = started_step.step_slug
241
- -- Generate task indices from 0 to initial_tasks-1
242
- CROSS JOIN LATERAL generate_series(0, started_step.initial_tasks - 1) AS task_idx(task_index)
243
- GROUP BY started_step.flow_slug, started_step.run_id, started_step.step_slug, step.opt_start_delay
244
- ),
245
- -- ---------- Send messages to queue ----------
246
- -- Uses batch sending for performance with large arrays
247
- sent_messages AS (
248
- SELECT
249
- mb.flow_slug,
250
- mb.run_id,
251
- mb.step_slug,
252
- task_indices.task_index,
253
- msg_ids.msg_id
254
- FROM message_batches mb
255
- CROSS JOIN LATERAL unnest(mb.task_indices) WITH ORDINALITY AS task_indices(task_index, idx_ord)
256
- CROSS JOIN LATERAL pgmq.send_batch(mb.flow_slug, mb.messages, mb.delay) WITH ORDINALITY AS msg_ids(msg_id, msg_ord)
257
- WHERE task_indices.idx_ord = msg_ids.msg_ord
258
- ),
259
-
260
- -- ---------- Broadcast step:started events ----------
261
- broadcast_events AS (
262
- SELECT
263
- realtime.send(
264
- jsonb_build_object(
265
- 'event_type', 'step:started',
266
- 'run_id', started_step.run_id,
267
- 'step_slug', started_step.step_slug,
268
- 'status', 'started',
269
- 'started_at', started_step.started_at,
270
- 'remaining_tasks', started_step.remaining_tasks,
271
- 'remaining_deps', started_step.remaining_deps
272
- ),
273
- concat('step:', started_step.step_slug, ':started'),
274
- concat('pgflow:run:', started_step.run_id),
275
- false
276
- )
277
- FROM started_step_states AS started_step
278
- )
279
-
280
- -- ==========================================
281
- -- RECORD TASKS IN DATABASE
282
- -- ==========================================
283
- INSERT INTO pgflow.step_tasks (flow_slug, run_id, step_slug, task_index, message_id)
284
- SELECT
285
- sent_messages.flow_slug,
286
- sent_messages.run_id,
287
- sent_messages.step_slug,
288
- sent_messages.task_index,
289
- sent_messages.msg_id
290
- FROM sent_messages;
291
-
292
- end;
293
- $$;
294
- -- Create "cascade_complete_taskless_steps" function
295
- CREATE FUNCTION "pgflow"."cascade_complete_taskless_steps" ("run_id" uuid) RETURNS integer LANGUAGE plpgsql AS $$
296
- DECLARE
297
- v_total_completed int := 0;
298
- v_iteration_completed int;
299
- v_iterations int := 0;
300
- v_max_iterations int := 50;
301
- BEGIN
302
- -- ==========================================
303
- -- ITERATIVE CASCADE COMPLETION
304
- -- ==========================================
305
- -- Completes taskless steps in waves until none remain
306
- LOOP
307
- -- ---------- Safety check ----------
308
- v_iterations := v_iterations + 1;
309
- IF v_iterations > v_max_iterations THEN
310
- RAISE EXCEPTION 'Cascade loop exceeded safety limit of % iterations', v_max_iterations;
311
- END IF;
312
-
313
- -- ==========================================
314
- -- COMPLETE READY TASKLESS STEPS
315
- -- ==========================================
316
- WITH completed AS (
317
- -- ---------- Complete taskless steps ----------
318
- -- Steps with initial_tasks=0 and no remaining deps
319
- UPDATE pgflow.step_states ss
320
- SET status = 'completed',
321
- started_at = now(),
322
- completed_at = now(),
323
- remaining_tasks = 0
324
- FROM pgflow.steps s
325
- WHERE ss.run_id = cascade_complete_taskless_steps.run_id
326
- AND ss.flow_slug = s.flow_slug
327
- AND ss.step_slug = s.step_slug
328
- AND ss.status = 'created'
329
- AND ss.remaining_deps = 0
330
- AND ss.initial_tasks = 0
331
- -- Process in topological order to ensure proper cascade
332
- RETURNING ss.*
333
- ),
334
- -- ---------- Update dependent steps ----------
335
- -- Propagate completion and empty arrays to dependents
336
- dep_updates AS (
337
- UPDATE pgflow.step_states ss
338
- SET remaining_deps = ss.remaining_deps - dep_count.count,
339
- -- If the dependent is a map step and its dependency completed with 0 tasks,
340
- -- set its initial_tasks to 0 as well
341
- initial_tasks = CASE
342
- WHEN s.step_type = 'map' AND dep_count.has_zero_tasks
343
- THEN 0 -- Empty array propagation
344
- ELSE ss.initial_tasks -- Keep existing value (including NULL)
345
- END
346
- FROM (
347
- -- Aggregate dependency updates per dependent step
348
- SELECT
349
- d.flow_slug,
350
- d.step_slug as dependent_slug,
351
- COUNT(*) as count,
352
- BOOL_OR(c.initial_tasks = 0) as has_zero_tasks
353
- FROM completed c
354
- JOIN pgflow.deps d ON d.flow_slug = c.flow_slug
355
- AND d.dep_slug = c.step_slug
356
- GROUP BY d.flow_slug, d.step_slug
357
- ) dep_count,
358
- pgflow.steps s
359
- WHERE ss.run_id = cascade_complete_taskless_steps.run_id
360
- AND ss.flow_slug = dep_count.flow_slug
361
- AND ss.step_slug = dep_count.dependent_slug
362
- AND s.flow_slug = ss.flow_slug
363
- AND s.step_slug = ss.step_slug
364
- ),
365
- -- ---------- Update run counters ----------
366
- -- Only decrement remaining_steps; let maybe_complete_run handle finalization
367
- run_updates AS (
368
- UPDATE pgflow.runs r
369
- SET remaining_steps = r.remaining_steps - c.completed_count
370
- FROM (SELECT COUNT(*) AS completed_count FROM completed) c
371
- WHERE r.run_id = cascade_complete_taskless_steps.run_id
372
- AND c.completed_count > 0
373
- )
374
- -- ---------- Check iteration results ----------
375
- SELECT COUNT(*) INTO v_iteration_completed FROM completed;
376
-
377
- EXIT WHEN v_iteration_completed = 0; -- No more steps to complete
378
- v_total_completed := v_total_completed + v_iteration_completed;
379
- END LOOP;
380
-
381
- RETURN v_total_completed;
382
- END;
383
- $$;
384
- -- Modify "complete_task" function
385
- CREATE OR REPLACE FUNCTION "pgflow"."complete_task" ("run_id" uuid, "step_slug" text, "task_index" integer, "output" jsonb) RETURNS SETOF "pgflow"."step_tasks" LANGUAGE plpgsql SET "search_path" = '' AS $$
386
- declare
387
- v_step_state pgflow.step_states%ROWTYPE;
388
- v_dependent_map_slug text;
389
- v_run_record pgflow.runs%ROWTYPE;
390
- v_step_record pgflow.step_states%ROWTYPE;
391
- begin
392
-
393
- -- ==========================================
394
- -- GUARD: No mutations on failed runs
395
- -- ==========================================
396
- IF EXISTS (SELECT 1 FROM pgflow.runs WHERE pgflow.runs.run_id = complete_task.run_id AND pgflow.runs.status = 'failed') THEN
397
- RETURN QUERY SELECT * FROM pgflow.step_tasks
398
- WHERE pgflow.step_tasks.run_id = complete_task.run_id
399
- AND pgflow.step_tasks.step_slug = complete_task.step_slug
400
- AND pgflow.step_tasks.task_index = complete_task.task_index;
401
- RETURN;
402
- END IF;
403
-
404
- -- ==========================================
405
- -- LOCK ACQUISITION AND TYPE VALIDATION
406
- -- ==========================================
407
- -- Acquire locks first to prevent race conditions
408
- SELECT * INTO v_run_record FROM pgflow.runs
409
- WHERE pgflow.runs.run_id = complete_task.run_id
410
- FOR UPDATE;
411
-
412
- SELECT * INTO v_step_record FROM pgflow.step_states
413
- WHERE pgflow.step_states.run_id = complete_task.run_id
414
- AND pgflow.step_states.step_slug = complete_task.step_slug
415
- FOR UPDATE;
416
-
417
- -- Check for type violations AFTER acquiring locks
418
- SELECT child_step.step_slug INTO v_dependent_map_slug
419
- FROM pgflow.deps dependency
420
- JOIN pgflow.steps child_step ON child_step.flow_slug = dependency.flow_slug
421
- AND child_step.step_slug = dependency.step_slug
422
- JOIN pgflow.steps parent_step ON parent_step.flow_slug = dependency.flow_slug
423
- AND parent_step.step_slug = dependency.dep_slug
424
- JOIN pgflow.step_states child_state ON child_state.flow_slug = child_step.flow_slug
425
- AND child_state.step_slug = child_step.step_slug
426
- WHERE dependency.dep_slug = complete_task.step_slug -- parent is the completing step
427
- AND dependency.flow_slug = v_run_record.flow_slug
428
- AND parent_step.step_type = 'single' -- Only validate single steps
429
- AND child_step.step_type = 'map'
430
- AND child_state.run_id = complete_task.run_id
431
- AND child_state.initial_tasks IS NULL
432
- AND (complete_task.output IS NULL OR jsonb_typeof(complete_task.output) != 'array')
433
- LIMIT 1;
434
-
435
- -- Handle type violation if detected
436
- IF v_dependent_map_slug IS NOT NULL THEN
437
- -- Mark run as failed immediately
438
- UPDATE pgflow.runs
439
- SET status = 'failed',
440
- failed_at = now()
441
- WHERE pgflow.runs.run_id = complete_task.run_id;
442
-
443
- -- Archive all active messages (both queued and started) to prevent orphaned messages
444
- PERFORM pgmq.archive(
445
- v_run_record.flow_slug,
446
- array_agg(st.message_id)
447
- )
448
- FROM pgflow.step_tasks st
449
- WHERE st.run_id = complete_task.run_id
450
- AND st.status IN ('queued', 'started')
451
- AND st.message_id IS NOT NULL
452
- HAVING count(*) > 0; -- Only call archive if there are messages to archive
453
-
454
- -- Mark current task as failed and store the output
455
- UPDATE pgflow.step_tasks
456
- SET status = 'failed',
457
- failed_at = now(),
458
- output = complete_task.output, -- Store the output that caused the violation
459
- error_message = '[TYPE_VIOLATION] Produced ' ||
460
- CASE WHEN complete_task.output IS NULL THEN 'null'
461
- ELSE jsonb_typeof(complete_task.output) END ||
462
- ' instead of array'
463
- WHERE pgflow.step_tasks.run_id = complete_task.run_id
464
- AND pgflow.step_tasks.step_slug = complete_task.step_slug
465
- AND pgflow.step_tasks.task_index = complete_task.task_index;
466
-
467
- -- Mark step state as failed
468
- UPDATE pgflow.step_states
469
- SET status = 'failed',
470
- failed_at = now(),
471
- error_message = '[TYPE_VIOLATION] Map step ' || v_dependent_map_slug ||
472
- ' expects array input but dependency ' || complete_task.step_slug ||
473
- ' produced ' || CASE WHEN complete_task.output IS NULL THEN 'null'
474
- ELSE jsonb_typeof(complete_task.output) END
475
- WHERE pgflow.step_states.run_id = complete_task.run_id
476
- AND pgflow.step_states.step_slug = complete_task.step_slug;
477
-
478
- -- Archive the current task's message (it was started, now failed)
479
- PERFORM pgmq.archive(
480
- v_run_record.flow_slug,
481
- st.message_id -- Single message, use scalar form
482
- )
483
- FROM pgflow.step_tasks st
484
- WHERE st.run_id = complete_task.run_id
485
- AND st.step_slug = complete_task.step_slug
486
- AND st.task_index = complete_task.task_index
487
- AND st.message_id IS NOT NULL;
488
-
489
- -- Return empty result
490
- RETURN QUERY SELECT * FROM pgflow.step_tasks WHERE false;
491
- RETURN;
492
- END IF;
493
-
494
- -- ==========================================
495
- -- MAIN CTE CHAIN: Update task and propagate changes
496
- -- ==========================================
497
- WITH
498
- -- ---------- Task completion ----------
499
- -- Update the task record with completion status and output
500
- task AS (
501
- UPDATE pgflow.step_tasks
502
- SET
503
- status = 'completed',
504
- completed_at = now(),
505
- output = complete_task.output
506
- WHERE pgflow.step_tasks.run_id = complete_task.run_id
507
- AND pgflow.step_tasks.step_slug = complete_task.step_slug
508
- AND pgflow.step_tasks.task_index = complete_task.task_index
509
- AND pgflow.step_tasks.status = 'started'
510
- RETURNING *
511
- ),
512
- -- ---------- Step state update ----------
513
- -- Decrement remaining_tasks and potentially mark step as completed
514
- step_state AS (
515
- UPDATE pgflow.step_states
516
- SET
517
- status = CASE
518
- WHEN pgflow.step_states.remaining_tasks = 1 THEN 'completed' -- Will be 0 after decrement
519
- ELSE 'started'
520
- END,
521
- completed_at = CASE
522
- WHEN pgflow.step_states.remaining_tasks = 1 THEN now() -- Will be 0 after decrement
523
- ELSE NULL
524
- END,
525
- remaining_tasks = pgflow.step_states.remaining_tasks - 1
526
- FROM task
527
- WHERE pgflow.step_states.run_id = complete_task.run_id
528
- AND pgflow.step_states.step_slug = complete_task.step_slug
529
- RETURNING pgflow.step_states.*
530
- ),
531
- -- ---------- Dependency resolution ----------
532
- -- Find all child steps that depend on the completed parent step (only if parent completed)
533
- child_steps AS (
534
- SELECT deps.step_slug AS child_step_slug
535
- FROM pgflow.deps deps
536
- JOIN step_state parent_state ON parent_state.status = 'completed' AND deps.flow_slug = parent_state.flow_slug
537
- WHERE deps.dep_slug = complete_task.step_slug -- dep_slug is the parent, step_slug is the child
538
- ORDER BY deps.step_slug -- Ensure consistent ordering
539
- ),
540
- -- ---------- Lock child steps ----------
541
- -- Acquire locks on all child steps before updating them
542
- child_steps_lock AS (
543
- SELECT * FROM pgflow.step_states
544
- WHERE pgflow.step_states.run_id = complete_task.run_id
545
- AND pgflow.step_states.step_slug IN (SELECT child_step_slug FROM child_steps)
546
- FOR UPDATE
547
- ),
548
- -- ---------- Update child steps ----------
549
- -- Decrement remaining_deps and resolve NULL initial_tasks for map steps
550
- child_steps_update AS (
551
- UPDATE pgflow.step_states child_state
552
- SET remaining_deps = child_state.remaining_deps - 1,
553
- -- Resolve NULL initial_tasks for child map steps
554
- -- This is where child maps learn their array size from the parent
555
- -- This CTE only runs when the parent step is complete (see child_steps JOIN)
556
- initial_tasks = CASE
557
- WHEN child_step.step_type = 'map' AND child_state.initial_tasks IS NULL THEN
558
- CASE
559
- WHEN parent_step.step_type = 'map' THEN
560
- -- Map->map: Count all completed tasks from parent map
561
- -- We add 1 because the current task is being completed in this transaction
562
- -- but isn't yet visible as 'completed' in the step_tasks table
563
- -- TODO: Refactor to use future column step_states.total_tasks
564
- -- Would eliminate the COUNT query and just use parent_state.total_tasks
565
- (SELECT COUNT(*)::int + 1
566
- FROM pgflow.step_tasks parent_tasks
567
- WHERE parent_tasks.run_id = complete_task.run_id
568
- AND parent_tasks.step_slug = complete_task.step_slug
569
- AND parent_tasks.status = 'completed'
570
- AND parent_tasks.task_index != complete_task.task_index)
571
- ELSE
572
- -- Single->map: Use output array length (single steps complete immediately)
573
- CASE
574
- WHEN complete_task.output IS NOT NULL
575
- AND jsonb_typeof(complete_task.output) = 'array' THEN
576
- jsonb_array_length(complete_task.output)
577
- ELSE NULL -- Keep NULL if not an array
578
- END
579
- END
580
- ELSE child_state.initial_tasks -- Keep existing value (including NULL)
581
- END
582
- FROM child_steps children
583
- JOIN pgflow.steps child_step ON child_step.flow_slug = (SELECT r.flow_slug FROM pgflow.runs r WHERE r.run_id = complete_task.run_id)
584
- AND child_step.step_slug = children.child_step_slug
585
- JOIN pgflow.steps parent_step ON parent_step.flow_slug = (SELECT r.flow_slug FROM pgflow.runs r WHERE r.run_id = complete_task.run_id)
586
- AND parent_step.step_slug = complete_task.step_slug
587
- WHERE child_state.run_id = complete_task.run_id
588
- AND child_state.step_slug = children.child_step_slug
589
- )
590
- -- ---------- Update run remaining_steps ----------
591
- -- Decrement the run's remaining_steps counter if step completed
592
- UPDATE pgflow.runs
593
- SET remaining_steps = pgflow.runs.remaining_steps - 1
594
- FROM step_state
595
- WHERE pgflow.runs.run_id = complete_task.run_id
596
- AND step_state.status = 'completed';
597
-
598
- -- ==========================================
599
- -- POST-COMPLETION ACTIONS
600
- -- ==========================================
601
-
602
- -- ---------- Get updated state for broadcasting ----------
603
- SELECT * INTO v_step_state FROM pgflow.step_states
604
- WHERE pgflow.step_states.run_id = complete_task.run_id AND pgflow.step_states.step_slug = complete_task.step_slug;
605
-
606
- -- ---------- Handle step completion ----------
607
- IF v_step_state.status = 'completed' THEN
608
- -- Cascade complete any taskless steps that are now ready
609
- PERFORM pgflow.cascade_complete_taskless_steps(complete_task.run_id);
610
-
611
- -- Broadcast step:completed event
612
- -- For map steps, aggregate all task outputs; for single steps, use the task output
613
- PERFORM realtime.send(
614
- jsonb_build_object(
615
- 'event_type', 'step:completed',
616
- 'run_id', complete_task.run_id,
617
- 'step_slug', complete_task.step_slug,
618
- 'status', 'completed',
619
- 'output', CASE
620
- WHEN (SELECT s.step_type FROM pgflow.steps s
621
- WHERE s.flow_slug = v_step_state.flow_slug
622
- AND s.step_slug = complete_task.step_slug) = 'map' THEN
623
- -- Aggregate all task outputs for map steps
624
- (SELECT COALESCE(jsonb_agg(st.output ORDER BY st.task_index), '[]'::jsonb)
625
- FROM pgflow.step_tasks st
626
- WHERE st.run_id = complete_task.run_id
627
- AND st.step_slug = complete_task.step_slug
628
- AND st.status = 'completed')
629
- ELSE
630
- -- Single step: use the individual task output
631
- complete_task.output
632
- END,
633
- 'completed_at', v_step_state.completed_at
634
- ),
635
- concat('step:', complete_task.step_slug, ':completed'),
636
- concat('pgflow:run:', complete_task.run_id),
637
- false
638
- );
639
- END IF;
640
-
641
- -- ---------- Archive completed task message ----------
642
- -- Move message from active queue to archive table
643
- PERFORM (
644
- WITH completed_tasks AS (
645
- SELECT r.flow_slug, st.message_id
646
- FROM pgflow.step_tasks st
647
- JOIN pgflow.runs r ON st.run_id = r.run_id
648
- WHERE st.run_id = complete_task.run_id
649
- AND st.step_slug = complete_task.step_slug
650
- AND st.task_index = complete_task.task_index
651
- AND st.status = 'completed'
652
- )
653
- SELECT pgmq.archive(ct.flow_slug, ct.message_id)
654
- FROM completed_tasks ct
655
- WHERE EXISTS (SELECT 1 FROM completed_tasks)
656
- );
657
-
658
- -- ---------- Trigger next steps ----------
659
- -- Start any steps that are now ready (deps satisfied)
660
- PERFORM pgflow.start_ready_steps(complete_task.run_id);
661
-
662
- -- Check if the entire run is complete
663
- PERFORM pgflow.maybe_complete_run(complete_task.run_id);
664
-
665
- -- ---------- Return completed task ----------
666
- RETURN QUERY SELECT *
667
- FROM pgflow.step_tasks AS step_task
668
- WHERE step_task.run_id = complete_task.run_id
669
- AND step_task.step_slug = complete_task.step_slug
670
- AND step_task.task_index = complete_task.task_index;
671
-
672
- end;
673
- $$;
674
- -- Modify "fail_task" function
675
- CREATE OR REPLACE FUNCTION "pgflow"."fail_task" ("run_id" uuid, "step_slug" text, "task_index" integer, "error_message" text) RETURNS SETOF "pgflow"."step_tasks" LANGUAGE plpgsql SET "search_path" = '' AS $$
676
- DECLARE
677
- v_run_failed boolean;
678
- v_step_failed boolean;
679
- begin
680
-
681
- -- If run is already failed, no retries allowed
682
- IF EXISTS (SELECT 1 FROM pgflow.runs WHERE pgflow.runs.run_id = fail_task.run_id AND pgflow.runs.status = 'failed') THEN
683
- UPDATE pgflow.step_tasks
684
- SET status = 'failed',
685
- failed_at = now(),
686
- error_message = fail_task.error_message
687
- WHERE pgflow.step_tasks.run_id = fail_task.run_id
688
- AND pgflow.step_tasks.step_slug = fail_task.step_slug
689
- AND pgflow.step_tasks.task_index = fail_task.task_index
690
- AND pgflow.step_tasks.status = 'started';
691
-
692
- -- Archive the task's message
693
- PERFORM pgmq.archive(r.flow_slug, ARRAY_AGG(st.message_id))
694
- FROM pgflow.step_tasks st
695
- JOIN pgflow.runs r ON st.run_id = r.run_id
696
- WHERE st.run_id = fail_task.run_id
697
- AND st.step_slug = fail_task.step_slug
698
- AND st.task_index = fail_task.task_index
699
- AND st.message_id IS NOT NULL
700
- GROUP BY r.flow_slug
701
- HAVING COUNT(st.message_id) > 0;
702
-
703
- RETURN QUERY SELECT * FROM pgflow.step_tasks
704
- WHERE pgflow.step_tasks.run_id = fail_task.run_id
705
- AND pgflow.step_tasks.step_slug = fail_task.step_slug
706
- AND pgflow.step_tasks.task_index = fail_task.task_index;
707
- RETURN;
708
- END IF;
709
-
710
- WITH run_lock AS (
711
- SELECT * FROM pgflow.runs
712
- WHERE pgflow.runs.run_id = fail_task.run_id
713
- FOR UPDATE
714
- ),
715
- step_lock AS (
716
- SELECT * FROM pgflow.step_states
717
- WHERE pgflow.step_states.run_id = fail_task.run_id
718
- AND pgflow.step_states.step_slug = fail_task.step_slug
719
- FOR UPDATE
720
- ),
721
- flow_info AS (
722
- SELECT r.flow_slug
723
- FROM pgflow.runs r
724
- WHERE r.run_id = fail_task.run_id
725
- ),
726
- config AS (
727
- SELECT
728
- COALESCE(s.opt_max_attempts, f.opt_max_attempts) AS opt_max_attempts,
729
- COALESCE(s.opt_base_delay, f.opt_base_delay) AS opt_base_delay
730
- FROM pgflow.steps s
731
- JOIN pgflow.flows f ON f.flow_slug = s.flow_slug
732
- JOIN flow_info fi ON fi.flow_slug = s.flow_slug
733
- WHERE s.flow_slug = fi.flow_slug AND s.step_slug = fail_task.step_slug
734
- ),
735
- fail_or_retry_task as (
736
- UPDATE pgflow.step_tasks as task
737
- SET
738
- status = CASE
739
- WHEN task.attempts_count < (SELECT opt_max_attempts FROM config) THEN 'queued'
740
- ELSE 'failed'
741
- END,
742
- failed_at = CASE
743
- WHEN task.attempts_count >= (SELECT opt_max_attempts FROM config) THEN now()
744
- ELSE NULL
745
- END,
746
- started_at = CASE
747
- WHEN task.attempts_count < (SELECT opt_max_attempts FROM config) THEN NULL
748
- ELSE task.started_at
749
- END,
750
- error_message = fail_task.error_message
751
- WHERE task.run_id = fail_task.run_id
752
- AND task.step_slug = fail_task.step_slug
753
- AND task.task_index = fail_task.task_index
754
- AND task.status = 'started'
755
- RETURNING *
756
- ),
757
- maybe_fail_step AS (
758
- UPDATE pgflow.step_states
759
- SET
760
- status = CASE
761
- WHEN (select fail_or_retry_task.status from fail_or_retry_task) = 'failed' THEN 'failed'
762
- ELSE pgflow.step_states.status
763
- END,
764
- failed_at = CASE
765
- WHEN (select fail_or_retry_task.status from fail_or_retry_task) = 'failed' THEN now()
766
- ELSE NULL
767
- END,
768
- error_message = CASE
769
- WHEN (select fail_or_retry_task.status from fail_or_retry_task) = 'failed' THEN fail_task.error_message
770
- ELSE NULL
771
- END
772
- FROM fail_or_retry_task
773
- WHERE pgflow.step_states.run_id = fail_task.run_id
774
- AND pgflow.step_states.step_slug = fail_task.step_slug
775
- RETURNING pgflow.step_states.*
776
- )
777
- -- Update run status
778
- UPDATE pgflow.runs
779
- SET status = CASE
780
- WHEN (select status from maybe_fail_step) = 'failed' THEN 'failed'
781
- ELSE status
782
- END,
783
- failed_at = CASE
784
- WHEN (select status from maybe_fail_step) = 'failed' THEN now()
785
- ELSE NULL
786
- END
787
- WHERE pgflow.runs.run_id = fail_task.run_id
788
- RETURNING (status = 'failed') INTO v_run_failed;
789
-
790
- -- Check if step failed by querying the step_states table
791
- SELECT (status = 'failed') INTO v_step_failed
792
- FROM pgflow.step_states
793
- WHERE pgflow.step_states.run_id = fail_task.run_id
794
- AND pgflow.step_states.step_slug = fail_task.step_slug;
795
-
796
- -- Send broadcast event for step failure if the step was failed
797
- IF v_step_failed THEN
798
- PERFORM realtime.send(
799
- jsonb_build_object(
800
- 'event_type', 'step:failed',
801
- 'run_id', fail_task.run_id,
802
- 'step_slug', fail_task.step_slug,
803
- 'status', 'failed',
804
- 'error_message', fail_task.error_message,
805
- 'failed_at', now()
806
- ),
807
- concat('step:', fail_task.step_slug, ':failed'),
808
- concat('pgflow:run:', fail_task.run_id),
809
- false
810
- );
811
- END IF;
812
-
813
- -- Send broadcast event for run failure if the run was failed
814
- IF v_run_failed THEN
815
- DECLARE
816
- v_flow_slug text;
817
- BEGIN
818
- SELECT flow_slug INTO v_flow_slug FROM pgflow.runs WHERE pgflow.runs.run_id = fail_task.run_id;
819
-
820
- PERFORM realtime.send(
821
- jsonb_build_object(
822
- 'event_type', 'run:failed',
823
- 'run_id', fail_task.run_id,
824
- 'flow_slug', v_flow_slug,
825
- 'status', 'failed',
826
- 'error_message', fail_task.error_message,
827
- 'failed_at', now()
828
- ),
829
- 'run:failed',
830
- concat('pgflow:run:', fail_task.run_id),
831
- false
832
- );
833
- END;
834
- END IF;
835
-
836
- -- Archive all active messages (both queued and started) when run fails
837
- IF v_run_failed THEN
838
- PERFORM pgmq.archive(r.flow_slug, ARRAY_AGG(st.message_id))
839
- FROM pgflow.step_tasks st
840
- JOIN pgflow.runs r ON st.run_id = r.run_id
841
- WHERE st.run_id = fail_task.run_id
842
- AND st.status IN ('queued', 'started')
843
- AND st.message_id IS NOT NULL
844
- GROUP BY r.flow_slug
845
- HAVING COUNT(st.message_id) > 0;
846
- END IF;
847
-
848
- -- For queued tasks: delay the message for retry with exponential backoff
849
- PERFORM (
850
- WITH retry_config AS (
851
- SELECT
852
- COALESCE(s.opt_base_delay, f.opt_base_delay) AS base_delay
853
- FROM pgflow.steps s
854
- JOIN pgflow.flows f ON f.flow_slug = s.flow_slug
855
- JOIN pgflow.runs r ON r.flow_slug = f.flow_slug
856
- WHERE r.run_id = fail_task.run_id
857
- AND s.step_slug = fail_task.step_slug
858
- ),
859
- queued_tasks AS (
860
- SELECT
861
- r.flow_slug,
862
- st.message_id,
863
- pgflow.calculate_retry_delay((SELECT base_delay FROM retry_config), st.attempts_count) AS calculated_delay
864
- FROM pgflow.step_tasks st
865
- JOIN pgflow.runs r ON st.run_id = r.run_id
866
- WHERE st.run_id = fail_task.run_id
867
- AND st.step_slug = fail_task.step_slug
868
- AND st.task_index = fail_task.task_index
869
- AND st.status = 'queued'
870
- )
871
- SELECT pgmq.set_vt(qt.flow_slug, qt.message_id, qt.calculated_delay)
872
- FROM queued_tasks qt
873
- WHERE EXISTS (SELECT 1 FROM queued_tasks)
874
- );
875
-
876
- -- For failed tasks: archive the message
877
- PERFORM pgmq.archive(r.flow_slug, ARRAY_AGG(st.message_id))
878
- FROM pgflow.step_tasks st
879
- JOIN pgflow.runs r ON st.run_id = r.run_id
880
- WHERE st.run_id = fail_task.run_id
881
- AND st.step_slug = fail_task.step_slug
882
- AND st.task_index = fail_task.task_index
883
- AND st.status = 'failed'
884
- AND st.message_id IS NOT NULL
885
- GROUP BY r.flow_slug
886
- HAVING COUNT(st.message_id) > 0;
887
-
888
- return query select *
889
- from pgflow.step_tasks st
890
- where st.run_id = fail_task.run_id
891
- and st.step_slug = fail_task.step_slug
892
- and st.task_index = fail_task.task_index;
893
-
894
- end;
895
- $$;
896
- -- Modify "start_flow" function
897
- CREATE OR REPLACE FUNCTION "pgflow"."start_flow" ("flow_slug" text, "input" jsonb, "run_id" uuid DEFAULT NULL::uuid) RETURNS SETOF "pgflow"."runs" LANGUAGE plpgsql SET "search_path" = '' AS $$
898
- declare
899
- v_created_run pgflow.runs%ROWTYPE;
900
- v_root_map_count int;
901
- begin
902
-
903
- -- ==========================================
904
- -- VALIDATION: Root map array input
905
- -- ==========================================
906
- WITH root_maps AS (
907
- SELECT step_slug
908
- FROM pgflow.steps
909
- WHERE steps.flow_slug = start_flow.flow_slug
910
- AND steps.step_type = 'map'
911
- AND steps.deps_count = 0
912
- )
913
- SELECT COUNT(*) INTO v_root_map_count FROM root_maps;
914
-
915
- -- If we have root map steps, validate that input is an array
916
- IF v_root_map_count > 0 THEN
917
- -- First check for NULL (should be caught by NOT NULL constraint, but be defensive)
918
- IF start_flow.input IS NULL THEN
919
- RAISE EXCEPTION 'Flow % has root map steps but input is NULL', start_flow.flow_slug;
920
- END IF;
921
-
922
- -- Then check if it's not an array
923
- IF jsonb_typeof(start_flow.input) != 'array' THEN
924
- RAISE EXCEPTION 'Flow % has root map steps but input is not an array (got %)',
925
- start_flow.flow_slug, jsonb_typeof(start_flow.input);
926
- END IF;
927
- END IF;
928
-
929
- -- ==========================================
930
- -- MAIN CTE CHAIN: Create run and step states
931
- -- ==========================================
932
- WITH
933
- -- ---------- Gather flow metadata ----------
934
- flow_steps AS (
935
- SELECT steps.flow_slug, steps.step_slug, steps.step_type, steps.deps_count
936
- FROM pgflow.steps
937
- WHERE steps.flow_slug = start_flow.flow_slug
938
- ),
939
- -- ---------- Create run record ----------
940
- created_run AS (
941
- INSERT INTO pgflow.runs (run_id, flow_slug, input, remaining_steps)
942
- VALUES (
943
- COALESCE(start_flow.run_id, gen_random_uuid()),
944
- start_flow.flow_slug,
945
- start_flow.input,
946
- (SELECT count(*) FROM flow_steps)
947
- )
948
- RETURNING *
949
- ),
950
- -- ---------- Create step states ----------
951
- -- Sets initial_tasks: known for root maps, NULL for dependent maps
952
- created_step_states AS (
953
- INSERT INTO pgflow.step_states (flow_slug, run_id, step_slug, remaining_deps, initial_tasks)
954
- SELECT
955
- fs.flow_slug,
956
- (SELECT created_run.run_id FROM created_run),
957
- fs.step_slug,
958
- fs.deps_count,
959
- -- Updated logic for initial_tasks:
960
- CASE
961
- WHEN fs.step_type = 'map' AND fs.deps_count = 0 THEN
962
- -- Root map: get array length from input
963
- CASE
964
- WHEN jsonb_typeof(start_flow.input) = 'array' THEN
965
- jsonb_array_length(start_flow.input)
966
- ELSE
967
- 1
968
- END
969
- WHEN fs.step_type = 'map' AND fs.deps_count > 0 THEN
970
- -- Dependent map: unknown until dependencies complete
971
- NULL
972
- ELSE
973
- -- Single steps: always 1 task
974
- 1
975
- END
976
- FROM flow_steps fs
977
- )
978
- SELECT * FROM created_run INTO v_created_run;
979
-
980
- -- ==========================================
981
- -- POST-CREATION ACTIONS
982
- -- ==========================================
983
-
984
- -- ---------- Broadcast run:started event ----------
985
- PERFORM realtime.send(
986
- jsonb_build_object(
987
- 'event_type', 'run:started',
988
- 'run_id', v_created_run.run_id,
989
- 'flow_slug', v_created_run.flow_slug,
990
- 'input', v_created_run.input,
991
- 'status', 'started',
992
- 'remaining_steps', v_created_run.remaining_steps,
993
- 'started_at', v_created_run.started_at
994
- ),
995
- 'run:started',
996
- concat('pgflow:run:', v_created_run.run_id),
997
- false
998
- );
999
-
1000
- -- ---------- Complete taskless steps ----------
1001
- -- Handle empty array maps that should auto-complete
1002
- PERFORM pgflow.cascade_complete_taskless_steps(v_created_run.run_id);
1003
-
1004
- -- ---------- Start initial steps ----------
1005
- -- Start root steps (those with no dependencies)
1006
- PERFORM pgflow.start_ready_steps(v_created_run.run_id);
1007
-
1008
- -- ---------- Check for run completion ----------
1009
- -- If cascade completed all steps (zero-task flows), finalize the run
1010
- PERFORM pgflow.maybe_complete_run(v_created_run.run_id);
1011
-
1012
- RETURN QUERY SELECT * FROM pgflow.runs where pgflow.runs.run_id = v_created_run.run_id;
1013
-
1014
- end;
1015
- $$;
1016
- -- Modify "start_tasks" function
1017
- CREATE OR REPLACE FUNCTION "pgflow"."start_tasks" ("flow_slug" text, "msg_ids" bigint[], "worker_id" uuid) RETURNS SETOF "pgflow"."step_task_record" LANGUAGE sql SET "search_path" = '' AS $$
1018
- with tasks as (
1019
- select
1020
- task.flow_slug,
1021
- task.run_id,
1022
- task.step_slug,
1023
- task.task_index,
1024
- task.message_id
1025
- from pgflow.step_tasks as task
1026
- join pgflow.runs r on r.run_id = task.run_id
1027
- where task.flow_slug = start_tasks.flow_slug
1028
- and task.message_id = any(msg_ids)
1029
- and task.status = 'queued'
1030
- -- MVP: Don't start tasks on failed runs
1031
- and r.status != 'failed'
1032
- ),
1033
- start_tasks_update as (
1034
- update pgflow.step_tasks
1035
- set
1036
- attempts_count = attempts_count + 1,
1037
- status = 'started',
1038
- started_at = now(),
1039
- last_worker_id = worker_id
1040
- from tasks
1041
- where step_tasks.message_id = tasks.message_id
1042
- and step_tasks.flow_slug = tasks.flow_slug
1043
- and step_tasks.status = 'queued'
1044
- ),
1045
- runs as (
1046
- select
1047
- r.run_id,
1048
- r.input
1049
- from pgflow.runs r
1050
- where r.run_id in (select run_id from tasks)
1051
- ),
1052
- deps as (
1053
- select
1054
- st.run_id,
1055
- st.step_slug,
1056
- dep.dep_slug,
1057
- -- Aggregate map outputs or use single output
1058
- CASE
1059
- WHEN dep_step.step_type = 'map' THEN
1060
- -- Aggregate all task outputs ordered by task_index
1061
- -- Use COALESCE to return empty array if no tasks
1062
- (SELECT COALESCE(jsonb_agg(dt.output ORDER BY dt.task_index), '[]'::jsonb)
1063
- FROM pgflow.step_tasks dt
1064
- WHERE dt.run_id = st.run_id
1065
- AND dt.step_slug = dep.dep_slug
1066
- AND dt.status = 'completed')
1067
- ELSE
1068
- -- Single step: use the single task output
1069
- dep_task.output
1070
- END as dep_output
1071
- from tasks st
1072
- join pgflow.deps dep on dep.flow_slug = st.flow_slug and dep.step_slug = st.step_slug
1073
- join pgflow.steps dep_step on dep_step.flow_slug = dep.flow_slug and dep_step.step_slug = dep.dep_slug
1074
- left join pgflow.step_tasks dep_task on
1075
- dep_task.run_id = st.run_id and
1076
- dep_task.step_slug = dep.dep_slug and
1077
- dep_task.status = 'completed'
1078
- and dep_step.step_type = 'single' -- Only join for single steps
1079
- ),
1080
- deps_outputs as (
1081
- select
1082
- d.run_id,
1083
- d.step_slug,
1084
- jsonb_object_agg(d.dep_slug, d.dep_output) as deps_output,
1085
- count(*) as dep_count
1086
- from deps d
1087
- group by d.run_id, d.step_slug
1088
- ),
1089
- timeouts as (
1090
- select
1091
- task.message_id,
1092
- task.flow_slug,
1093
- coalesce(step.opt_timeout, flow.opt_timeout) + 2 as vt_delay
1094
- from tasks task
1095
- join pgflow.flows flow on flow.flow_slug = task.flow_slug
1096
- join pgflow.steps step on step.flow_slug = task.flow_slug and step.step_slug = task.step_slug
1097
- ),
1098
- -- Batch update visibility timeouts for all messages
1099
- set_vt_batch as (
1100
- select pgflow.set_vt_batch(
1101
- start_tasks.flow_slug,
1102
- array_agg(t.message_id order by t.message_id),
1103
- array_agg(t.vt_delay order by t.message_id)
1104
- )
1105
- from timeouts t
1106
- )
1107
- select
1108
- st.flow_slug,
1109
- st.run_id,
1110
- st.step_slug,
1111
- -- ==========================================
1112
- -- INPUT CONSTRUCTION LOGIC
1113
- -- ==========================================
1114
- -- This nested CASE statement determines how to construct the input
1115
- -- for each task based on the step type (map vs non-map).
1116
- --
1117
- -- The fundamental difference:
1118
- -- - Map steps: Receive RAW array elements (e.g., just 42 or "hello")
1119
- -- - Non-map steps: Receive structured objects with named keys
1120
- -- (e.g., {"run": {...}, "dependency1": {...}})
1121
- -- ==========================================
1122
- CASE
1123
- -- -------------------- MAP STEPS --------------------
1124
- -- Map steps process arrays element-by-element.
1125
- -- Each task receives ONE element from the array at its task_index position.
1126
- WHEN step.step_type = 'map' THEN
1127
- -- Map steps get raw array elements without any wrapper object
1128
- CASE
1129
- -- ROOT MAP: Gets array from run input
1130
- -- Example: run input = [1, 2, 3]
1131
- -- task 0 gets: 1
1132
- -- task 1 gets: 2
1133
- -- task 2 gets: 3
1134
- WHEN step.deps_count = 0 THEN
1135
- -- Root map (deps_count = 0): no dependencies, reads from run input.
1136
- -- Extract the element at task_index from the run's input array.
1137
- -- Note: If run input is not an array, this will return NULL
1138
- -- and the flow will fail (validated in start_flow).
1139
- jsonb_array_element(r.input, st.task_index)
1140
-
1141
- -- DEPENDENT MAP: Gets array from its single dependency
1142
- -- Example: dependency output = ["a", "b", "c"]
1143
- -- task 0 gets: "a"
1144
- -- task 1 gets: "b"
1145
- -- task 2 gets: "c"
1146
- ELSE
1147
- -- Has dependencies (should be exactly 1 for map steps).
1148
- -- Extract the element at task_index from the dependency's output array.
1149
- --
1150
- -- Why the subquery with jsonb_each?
1151
- -- - The dependency outputs a raw array: [1, 2, 3]
1152
- -- - deps_outputs aggregates it into: {"dep_name": [1, 2, 3]}
1153
- -- - We need to unwrap and get just the array value
1154
- -- - Map steps have exactly 1 dependency (enforced by add_step)
1155
- -- - So jsonb_each will return exactly 1 row
1156
- -- - We extract the 'value' which is the raw array [1, 2, 3]
1157
- -- - Then get the element at task_index from that array
1158
- (SELECT jsonb_array_element(value, st.task_index)
1159
- FROM jsonb_each(dep_out.deps_output)
1160
- LIMIT 1)
1161
- END
1162
-
1163
- -- -------------------- NON-MAP STEPS --------------------
1164
- -- Regular (non-map) steps receive ALL inputs as a structured object.
1165
- -- This includes the original run input plus all dependency outputs.
1166
- ELSE
1167
- -- Non-map steps get structured input with named keys
1168
- -- Example output: {
1169
- -- "run": {"original": "input"},
1170
- -- "step1": {"output": "from_step1"},
1171
- -- "step2": {"output": "from_step2"}
1172
- -- }
1173
- --
1174
- -- Build object with 'run' key containing original input
1175
- jsonb_build_object('run', r.input) ||
1176
- -- Merge with deps_output which already has dependency outputs
1177
- -- deps_output format: {"dep1": output1, "dep2": output2, ...}
1178
- -- If no dependencies, defaults to empty object
1179
- coalesce(dep_out.deps_output, '{}'::jsonb)
1180
- END as input,
1181
- st.message_id as msg_id,
1182
- st.task_index as task_index
1183
- from tasks st
1184
- join runs r on st.run_id = r.run_id
1185
- join pgflow.steps step on
1186
- step.flow_slug = st.flow_slug and
1187
- step.step_slug = st.step_slug
1188
- left join deps_outputs dep_out on
1189
- dep_out.run_id = st.run_id and
1190
- dep_out.step_slug = st.step_slug
1191
- $$;
1192
- -- Create "add_step" function
1193
- CREATE FUNCTION "pgflow"."add_step" ("flow_slug" text, "step_slug" text, "deps_slugs" text[] DEFAULT '{}', "max_attempts" integer DEFAULT NULL::integer, "base_delay" integer DEFAULT NULL::integer, "timeout" integer DEFAULT NULL::integer, "start_delay" integer DEFAULT NULL::integer, "step_type" text DEFAULT 'single') RETURNS "pgflow"."steps" LANGUAGE plpgsql SET "search_path" = '' AS $$
1194
- DECLARE
1195
- result_step pgflow.steps;
1196
- next_idx int;
1197
- BEGIN
1198
- -- Validate map step constraints
1199
- -- Map steps can have either:
1200
- -- 0 dependencies (root map - maps over flow input array)
1201
- -- 1 dependency (dependent map - maps over dependency output array)
1202
- IF COALESCE(add_step.step_type, 'single') = 'map' AND COALESCE(array_length(add_step.deps_slugs, 1), 0) > 1 THEN
1203
- RAISE EXCEPTION 'Map step "%" can have at most one dependency, but % were provided: %',
1204
- add_step.step_slug,
1205
- COALESCE(array_length(add_step.deps_slugs, 1), 0),
1206
- array_to_string(add_step.deps_slugs, ', ');
1207
- END IF;
1208
-
1209
- -- Get next step index
1210
- SELECT COALESCE(MAX(s.step_index) + 1, 0) INTO next_idx
1211
- FROM pgflow.steps s
1212
- WHERE s.flow_slug = add_step.flow_slug;
1213
-
1214
- -- Create the step
1215
- INSERT INTO pgflow.steps (
1216
- flow_slug, step_slug, step_type, step_index, deps_count,
1217
- opt_max_attempts, opt_base_delay, opt_timeout, opt_start_delay
1218
- )
1219
- VALUES (
1220
- add_step.flow_slug,
1221
- add_step.step_slug,
1222
- COALESCE(add_step.step_type, 'single'),
1223
- next_idx,
1224
- COALESCE(array_length(add_step.deps_slugs, 1), 0),
1225
- add_step.max_attempts,
1226
- add_step.base_delay,
1227
- add_step.timeout,
1228
- add_step.start_delay
1229
- )
1230
- ON CONFLICT ON CONSTRAINT steps_pkey
1231
- DO UPDATE SET step_slug = EXCLUDED.step_slug
1232
- RETURNING * INTO result_step;
1233
-
1234
- -- Insert dependencies
1235
- INSERT INTO pgflow.deps (flow_slug, dep_slug, step_slug)
1236
- SELECT add_step.flow_slug, d.dep_slug, add_step.step_slug
1237
- FROM unnest(COALESCE(add_step.deps_slugs, '{}')) AS d(dep_slug)
1238
- WHERE add_step.deps_slugs IS NOT NULL AND array_length(add_step.deps_slugs, 1) > 0
1239
- ON CONFLICT ON CONSTRAINT deps_pkey DO NOTHING;
1240
-
1241
- RETURN result_step;
1242
- END;
1243
- $$;
1244
- -- Drop "add_step" function
1245
- DROP FUNCTION "pgflow"."add_step" (text, text, integer, integer, integer, integer);
1246
- -- Drop "add_step" function
1247
- DROP FUNCTION "pgflow"."add_step" (text, text, text[], integer, integer, integer, integer);
1248
- `,
1249
- };