@pgflow/core 0.0.0-array-map-steps-302d00a8-20250925065142 → 0.0.0-array-map-steps-b956f8f9-20251006084236
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +148 -72
- package/dist/CHANGELOG.md +17 -15
- package/dist/README.md +148 -72
- package/dist/package.json +1 -1
- package/dist/supabase/migrations/{20250919101802_pgflow_temp_orphaned_messages_index.sql → 20251006073122_pgflow_add_map_step_type.sql} +533 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
- package/dist/PLAN_race_condition_testing.md +0 -176
- package/dist/supabase/migrations/20250912075001_pgflow_temp_pr1_schema.sql +0 -185
- package/dist/supabase/migrations/20250912080800_pgflow_temp_pr2_root_maps.sql +0 -95
- package/dist/supabase/migrations/20250912125339_pgflow_TEMP_task_spawning_optimization.sql +0 -146
- package/dist/supabase/migrations/20250916093518_pgflow_temp_add_cascade_complete.sql +0 -321
- package/dist/supabase/migrations/20250916142327_pgflow_temp_make_initial_tasks_nullable.sql +0 -624
- package/dist/supabase/migrations/20250916203905_pgflow_temp_handle_arrays_in_start_tasks.sql +0 -157
- package/dist/supabase/migrations/20250918042753_pgflow_temp_handle_map_output_aggregation.sql +0 -489
- package/dist/supabase/migrations/20250919135211_pgflow_temp_return_task_index_in_start_tasks.sql +0 -178
|
@@ -1,5 +1,95 @@
|
|
|
1
|
+
-- Modify "step_task_record" composite type
|
|
2
|
+
ALTER TYPE "pgflow"."step_task_record" ADD ATTRIBUTE "task_index" integer;
|
|
3
|
+
-- Modify "step_states" table
|
|
4
|
+
ALTER TABLE "pgflow"."step_states" DROP CONSTRAINT "step_states_remaining_tasks_check", ADD CONSTRAINT "initial_tasks_known_when_started" CHECK ((status <> 'started'::text) OR (initial_tasks IS NOT NULL)), ADD CONSTRAINT "remaining_tasks_state_consistency" CHECK ((remaining_tasks IS NULL) OR (status <> 'created'::text)), ADD CONSTRAINT "step_states_initial_tasks_check" CHECK ((initial_tasks IS NULL) OR (initial_tasks >= 0)), ALTER COLUMN "remaining_tasks" DROP NOT NULL, ALTER COLUMN "remaining_tasks" DROP DEFAULT, ADD COLUMN "initial_tasks" integer NULL;
|
|
1
5
|
-- Modify "step_tasks" table
|
|
2
|
-
ALTER TABLE "pgflow"."step_tasks" DROP CONSTRAINT "output_valid_only_for_completed", ADD CONSTRAINT "output_valid_only_for_completed" CHECK ((output IS NULL) OR (status = ANY (ARRAY['completed'::text, 'failed'::text])));
|
|
6
|
+
ALTER TABLE "pgflow"."step_tasks" DROP CONSTRAINT "only_single_task_per_step", DROP CONSTRAINT "output_valid_only_for_completed", ADD CONSTRAINT "output_valid_only_for_completed" CHECK ((output IS NULL) OR (status = ANY (ARRAY['completed'::text, 'failed'::text])));
|
|
7
|
+
-- Modify "steps" table
|
|
8
|
+
ALTER TABLE "pgflow"."steps" DROP CONSTRAINT "steps_step_type_check", ADD CONSTRAINT "steps_step_type_check" CHECK (step_type = ANY (ARRAY['single'::text, 'map'::text]));
|
|
9
|
+
-- Modify "maybe_complete_run" function
|
|
10
|
+
CREATE OR REPLACE FUNCTION "pgflow"."maybe_complete_run" ("run_id" uuid) RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
11
|
+
declare
|
|
12
|
+
v_completed_run pgflow.runs%ROWTYPE;
|
|
13
|
+
begin
|
|
14
|
+
-- ==========================================
|
|
15
|
+
-- CHECK AND COMPLETE RUN IF FINISHED
|
|
16
|
+
-- ==========================================
|
|
17
|
+
-- ---------- Complete run if all steps done ----------
|
|
18
|
+
UPDATE pgflow.runs
|
|
19
|
+
SET
|
|
20
|
+
status = 'completed',
|
|
21
|
+
completed_at = now(),
|
|
22
|
+
-- Only compute expensive aggregation when actually completing the run
|
|
23
|
+
output = (
|
|
24
|
+
-- ---------- Gather outputs from leaf steps ----------
|
|
25
|
+
-- Leaf steps = steps with no dependents
|
|
26
|
+
-- For map steps: aggregate all task outputs into array
|
|
27
|
+
-- For single steps: use the single task output
|
|
28
|
+
SELECT jsonb_object_agg(
|
|
29
|
+
step_slug,
|
|
30
|
+
CASE
|
|
31
|
+
WHEN step_type = 'map' THEN aggregated_output
|
|
32
|
+
ELSE single_output
|
|
33
|
+
END
|
|
34
|
+
)
|
|
35
|
+
FROM (
|
|
36
|
+
SELECT DISTINCT
|
|
37
|
+
leaf_state.step_slug,
|
|
38
|
+
leaf_step.step_type,
|
|
39
|
+
-- For map steps: aggregate all task outputs
|
|
40
|
+
CASE WHEN leaf_step.step_type = 'map' THEN
|
|
41
|
+
(SELECT COALESCE(jsonb_agg(leaf_task.output ORDER BY leaf_task.task_index), '[]'::jsonb)
|
|
42
|
+
FROM pgflow.step_tasks leaf_task
|
|
43
|
+
WHERE leaf_task.run_id = leaf_state.run_id
|
|
44
|
+
AND leaf_task.step_slug = leaf_state.step_slug
|
|
45
|
+
AND leaf_task.status = 'completed')
|
|
46
|
+
END as aggregated_output,
|
|
47
|
+
-- For single steps: get the single output
|
|
48
|
+
CASE WHEN leaf_step.step_type = 'single' THEN
|
|
49
|
+
(SELECT leaf_task.output
|
|
50
|
+
FROM pgflow.step_tasks leaf_task
|
|
51
|
+
WHERE leaf_task.run_id = leaf_state.run_id
|
|
52
|
+
AND leaf_task.step_slug = leaf_state.step_slug
|
|
53
|
+
AND leaf_task.status = 'completed'
|
|
54
|
+
LIMIT 1)
|
|
55
|
+
END as single_output
|
|
56
|
+
FROM pgflow.step_states leaf_state
|
|
57
|
+
JOIN pgflow.steps leaf_step ON leaf_step.flow_slug = leaf_state.flow_slug AND leaf_step.step_slug = leaf_state.step_slug
|
|
58
|
+
WHERE leaf_state.run_id = maybe_complete_run.run_id
|
|
59
|
+
AND leaf_state.status = 'completed'
|
|
60
|
+
AND NOT EXISTS (
|
|
61
|
+
SELECT 1
|
|
62
|
+
FROM pgflow.deps dep
|
|
63
|
+
WHERE dep.flow_slug = leaf_state.flow_slug
|
|
64
|
+
AND dep.dep_slug = leaf_state.step_slug
|
|
65
|
+
)
|
|
66
|
+
) leaf_outputs
|
|
67
|
+
)
|
|
68
|
+
WHERE pgflow.runs.run_id = maybe_complete_run.run_id
|
|
69
|
+
AND pgflow.runs.remaining_steps = 0
|
|
70
|
+
AND pgflow.runs.status != 'completed'
|
|
71
|
+
RETURNING * INTO v_completed_run;
|
|
72
|
+
|
|
73
|
+
-- ==========================================
|
|
74
|
+
-- BROADCAST COMPLETION EVENT
|
|
75
|
+
-- ==========================================
|
|
76
|
+
IF v_completed_run.run_id IS NOT NULL THEN
|
|
77
|
+
PERFORM realtime.send(
|
|
78
|
+
jsonb_build_object(
|
|
79
|
+
'event_type', 'run:completed',
|
|
80
|
+
'run_id', v_completed_run.run_id,
|
|
81
|
+
'flow_slug', v_completed_run.flow_slug,
|
|
82
|
+
'status', 'completed',
|
|
83
|
+
'output', v_completed_run.output,
|
|
84
|
+
'completed_at', v_completed_run.completed_at
|
|
85
|
+
),
|
|
86
|
+
'run:completed',
|
|
87
|
+
concat('pgflow:run:', v_completed_run.run_id),
|
|
88
|
+
false
|
|
89
|
+
);
|
|
90
|
+
END IF;
|
|
91
|
+
end;
|
|
92
|
+
$$;
|
|
3
93
|
-- Modify "start_ready_steps" function
|
|
4
94
|
CREATE OR REPLACE FUNCTION "pgflow"."start_ready_steps" ("run_id" uuid) RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
5
95
|
begin
|
|
@@ -174,6 +264,96 @@ FROM sent_messages;
|
|
|
174
264
|
|
|
175
265
|
end;
|
|
176
266
|
$$;
|
|
267
|
+
-- Create "cascade_complete_taskless_steps" function
|
|
268
|
+
CREATE FUNCTION "pgflow"."cascade_complete_taskless_steps" ("run_id" uuid) RETURNS integer LANGUAGE plpgsql AS $$
|
|
269
|
+
DECLARE
|
|
270
|
+
v_total_completed int := 0;
|
|
271
|
+
v_iteration_completed int;
|
|
272
|
+
v_iterations int := 0;
|
|
273
|
+
v_max_iterations int := 50;
|
|
274
|
+
BEGIN
|
|
275
|
+
-- ==========================================
|
|
276
|
+
-- ITERATIVE CASCADE COMPLETION
|
|
277
|
+
-- ==========================================
|
|
278
|
+
-- Completes taskless steps in waves until none remain
|
|
279
|
+
LOOP
|
|
280
|
+
-- ---------- Safety check ----------
|
|
281
|
+
v_iterations := v_iterations + 1;
|
|
282
|
+
IF v_iterations > v_max_iterations THEN
|
|
283
|
+
RAISE EXCEPTION 'Cascade loop exceeded safety limit of % iterations', v_max_iterations;
|
|
284
|
+
END IF;
|
|
285
|
+
|
|
286
|
+
-- ==========================================
|
|
287
|
+
-- COMPLETE READY TASKLESS STEPS
|
|
288
|
+
-- ==========================================
|
|
289
|
+
WITH completed AS (
|
|
290
|
+
-- ---------- Complete taskless steps ----------
|
|
291
|
+
-- Steps with initial_tasks=0 and no remaining deps
|
|
292
|
+
UPDATE pgflow.step_states ss
|
|
293
|
+
SET status = 'completed',
|
|
294
|
+
started_at = now(),
|
|
295
|
+
completed_at = now(),
|
|
296
|
+
remaining_tasks = 0
|
|
297
|
+
FROM pgflow.steps s
|
|
298
|
+
WHERE ss.run_id = cascade_complete_taskless_steps.run_id
|
|
299
|
+
AND ss.flow_slug = s.flow_slug
|
|
300
|
+
AND ss.step_slug = s.step_slug
|
|
301
|
+
AND ss.status = 'created'
|
|
302
|
+
AND ss.remaining_deps = 0
|
|
303
|
+
AND ss.initial_tasks = 0
|
|
304
|
+
-- Process in topological order to ensure proper cascade
|
|
305
|
+
RETURNING ss.*
|
|
306
|
+
),
|
|
307
|
+
-- ---------- Update dependent steps ----------
|
|
308
|
+
-- Propagate completion and empty arrays to dependents
|
|
309
|
+
dep_updates AS (
|
|
310
|
+
UPDATE pgflow.step_states ss
|
|
311
|
+
SET remaining_deps = ss.remaining_deps - dep_count.count,
|
|
312
|
+
-- If the dependent is a map step and its dependency completed with 0 tasks,
|
|
313
|
+
-- set its initial_tasks to 0 as well
|
|
314
|
+
initial_tasks = CASE
|
|
315
|
+
WHEN s.step_type = 'map' AND dep_count.has_zero_tasks
|
|
316
|
+
THEN 0 -- Empty array propagation
|
|
317
|
+
ELSE ss.initial_tasks -- Keep existing value (including NULL)
|
|
318
|
+
END
|
|
319
|
+
FROM (
|
|
320
|
+
-- Aggregate dependency updates per dependent step
|
|
321
|
+
SELECT
|
|
322
|
+
d.flow_slug,
|
|
323
|
+
d.step_slug as dependent_slug,
|
|
324
|
+
COUNT(*) as count,
|
|
325
|
+
BOOL_OR(c.initial_tasks = 0) as has_zero_tasks
|
|
326
|
+
FROM completed c
|
|
327
|
+
JOIN pgflow.deps d ON d.flow_slug = c.flow_slug
|
|
328
|
+
AND d.dep_slug = c.step_slug
|
|
329
|
+
GROUP BY d.flow_slug, d.step_slug
|
|
330
|
+
) dep_count,
|
|
331
|
+
pgflow.steps s
|
|
332
|
+
WHERE ss.run_id = cascade_complete_taskless_steps.run_id
|
|
333
|
+
AND ss.flow_slug = dep_count.flow_slug
|
|
334
|
+
AND ss.step_slug = dep_count.dependent_slug
|
|
335
|
+
AND s.flow_slug = ss.flow_slug
|
|
336
|
+
AND s.step_slug = ss.step_slug
|
|
337
|
+
),
|
|
338
|
+
-- ---------- Update run counters ----------
|
|
339
|
+
-- Only decrement remaining_steps; let maybe_complete_run handle finalization
|
|
340
|
+
run_updates AS (
|
|
341
|
+
UPDATE pgflow.runs r
|
|
342
|
+
SET remaining_steps = r.remaining_steps - c.completed_count
|
|
343
|
+
FROM (SELECT COUNT(*) AS completed_count FROM completed) c
|
|
344
|
+
WHERE r.run_id = cascade_complete_taskless_steps.run_id
|
|
345
|
+
AND c.completed_count > 0
|
|
346
|
+
)
|
|
347
|
+
-- ---------- Check iteration results ----------
|
|
348
|
+
SELECT COUNT(*) INTO v_iteration_completed FROM completed;
|
|
349
|
+
|
|
350
|
+
EXIT WHEN v_iteration_completed = 0; -- No more steps to complete
|
|
351
|
+
v_total_completed := v_total_completed + v_iteration_completed;
|
|
352
|
+
END LOOP;
|
|
353
|
+
|
|
354
|
+
RETURN v_total_completed;
|
|
355
|
+
END;
|
|
356
|
+
$$;
|
|
177
357
|
-- Modify "complete_task" function
|
|
178
358
|
CREATE OR REPLACE FUNCTION "pgflow"."complete_task" ("run_id" uuid, "step_slug" text, "task_index" integer, "output" jsonb) RETURNS SETOF "pgflow"."step_tasks" LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
179
359
|
declare
|
|
@@ -686,3 +866,355 @@ where st.run_id = fail_task.run_id
|
|
|
686
866
|
|
|
687
867
|
end;
|
|
688
868
|
$$;
|
|
869
|
+
-- Modify "start_flow" function
|
|
870
|
+
CREATE OR REPLACE FUNCTION "pgflow"."start_flow" ("flow_slug" text, "input" jsonb, "run_id" uuid DEFAULT NULL::uuid) RETURNS SETOF "pgflow"."runs" LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
871
|
+
declare
|
|
872
|
+
v_created_run pgflow.runs%ROWTYPE;
|
|
873
|
+
v_root_map_count int;
|
|
874
|
+
begin
|
|
875
|
+
|
|
876
|
+
-- ==========================================
|
|
877
|
+
-- VALIDATION: Root map array input
|
|
878
|
+
-- ==========================================
|
|
879
|
+
WITH root_maps AS (
|
|
880
|
+
SELECT step_slug
|
|
881
|
+
FROM pgflow.steps
|
|
882
|
+
WHERE steps.flow_slug = start_flow.flow_slug
|
|
883
|
+
AND steps.step_type = 'map'
|
|
884
|
+
AND steps.deps_count = 0
|
|
885
|
+
)
|
|
886
|
+
SELECT COUNT(*) INTO v_root_map_count FROM root_maps;
|
|
887
|
+
|
|
888
|
+
-- If we have root map steps, validate that input is an array
|
|
889
|
+
IF v_root_map_count > 0 THEN
|
|
890
|
+
-- First check for NULL (should be caught by NOT NULL constraint, but be defensive)
|
|
891
|
+
IF start_flow.input IS NULL THEN
|
|
892
|
+
RAISE EXCEPTION 'Flow % has root map steps but input is NULL', start_flow.flow_slug;
|
|
893
|
+
END IF;
|
|
894
|
+
|
|
895
|
+
-- Then check if it's not an array
|
|
896
|
+
IF jsonb_typeof(start_flow.input) != 'array' THEN
|
|
897
|
+
RAISE EXCEPTION 'Flow % has root map steps but input is not an array (got %)',
|
|
898
|
+
start_flow.flow_slug, jsonb_typeof(start_flow.input);
|
|
899
|
+
END IF;
|
|
900
|
+
END IF;
|
|
901
|
+
|
|
902
|
+
-- ==========================================
|
|
903
|
+
-- MAIN CTE CHAIN: Create run and step states
|
|
904
|
+
-- ==========================================
|
|
905
|
+
WITH
|
|
906
|
+
-- ---------- Gather flow metadata ----------
|
|
907
|
+
flow_steps AS (
|
|
908
|
+
SELECT steps.flow_slug, steps.step_slug, steps.step_type, steps.deps_count
|
|
909
|
+
FROM pgflow.steps
|
|
910
|
+
WHERE steps.flow_slug = start_flow.flow_slug
|
|
911
|
+
),
|
|
912
|
+
-- ---------- Create run record ----------
|
|
913
|
+
created_run AS (
|
|
914
|
+
INSERT INTO pgflow.runs (run_id, flow_slug, input, remaining_steps)
|
|
915
|
+
VALUES (
|
|
916
|
+
COALESCE(start_flow.run_id, gen_random_uuid()),
|
|
917
|
+
start_flow.flow_slug,
|
|
918
|
+
start_flow.input,
|
|
919
|
+
(SELECT count(*) FROM flow_steps)
|
|
920
|
+
)
|
|
921
|
+
RETURNING *
|
|
922
|
+
),
|
|
923
|
+
-- ---------- Create step states ----------
|
|
924
|
+
-- Sets initial_tasks: known for root maps, NULL for dependent maps
|
|
925
|
+
created_step_states AS (
|
|
926
|
+
INSERT INTO pgflow.step_states (flow_slug, run_id, step_slug, remaining_deps, initial_tasks)
|
|
927
|
+
SELECT
|
|
928
|
+
fs.flow_slug,
|
|
929
|
+
(SELECT created_run.run_id FROM created_run),
|
|
930
|
+
fs.step_slug,
|
|
931
|
+
fs.deps_count,
|
|
932
|
+
-- Updated logic for initial_tasks:
|
|
933
|
+
CASE
|
|
934
|
+
WHEN fs.step_type = 'map' AND fs.deps_count = 0 THEN
|
|
935
|
+
-- Root map: get array length from input
|
|
936
|
+
CASE
|
|
937
|
+
WHEN jsonb_typeof(start_flow.input) = 'array' THEN
|
|
938
|
+
jsonb_array_length(start_flow.input)
|
|
939
|
+
ELSE
|
|
940
|
+
1
|
|
941
|
+
END
|
|
942
|
+
WHEN fs.step_type = 'map' AND fs.deps_count > 0 THEN
|
|
943
|
+
-- Dependent map: unknown until dependencies complete
|
|
944
|
+
NULL
|
|
945
|
+
ELSE
|
|
946
|
+
-- Single steps: always 1 task
|
|
947
|
+
1
|
|
948
|
+
END
|
|
949
|
+
FROM flow_steps fs
|
|
950
|
+
)
|
|
951
|
+
SELECT * FROM created_run INTO v_created_run;
|
|
952
|
+
|
|
953
|
+
-- ==========================================
|
|
954
|
+
-- POST-CREATION ACTIONS
|
|
955
|
+
-- ==========================================
|
|
956
|
+
|
|
957
|
+
-- ---------- Broadcast run:started event ----------
|
|
958
|
+
PERFORM realtime.send(
|
|
959
|
+
jsonb_build_object(
|
|
960
|
+
'event_type', 'run:started',
|
|
961
|
+
'run_id', v_created_run.run_id,
|
|
962
|
+
'flow_slug', v_created_run.flow_slug,
|
|
963
|
+
'input', v_created_run.input,
|
|
964
|
+
'status', 'started',
|
|
965
|
+
'remaining_steps', v_created_run.remaining_steps,
|
|
966
|
+
'started_at', v_created_run.started_at
|
|
967
|
+
),
|
|
968
|
+
'run:started',
|
|
969
|
+
concat('pgflow:run:', v_created_run.run_id),
|
|
970
|
+
false
|
|
971
|
+
);
|
|
972
|
+
|
|
973
|
+
-- ---------- Complete taskless steps ----------
|
|
974
|
+
-- Handle empty array maps that should auto-complete
|
|
975
|
+
PERFORM pgflow.cascade_complete_taskless_steps(v_created_run.run_id);
|
|
976
|
+
|
|
977
|
+
-- ---------- Start initial steps ----------
|
|
978
|
+
-- Start root steps (those with no dependencies)
|
|
979
|
+
PERFORM pgflow.start_ready_steps(v_created_run.run_id);
|
|
980
|
+
|
|
981
|
+
-- ---------- Check for run completion ----------
|
|
982
|
+
-- If cascade completed all steps (zero-task flows), finalize the run
|
|
983
|
+
PERFORM pgflow.maybe_complete_run(v_created_run.run_id);
|
|
984
|
+
|
|
985
|
+
RETURN QUERY SELECT * FROM pgflow.runs where pgflow.runs.run_id = v_created_run.run_id;
|
|
986
|
+
|
|
987
|
+
end;
|
|
988
|
+
$$;
|
|
989
|
+
-- Modify "start_tasks" function
|
|
990
|
+
CREATE OR REPLACE FUNCTION "pgflow"."start_tasks" ("flow_slug" text, "msg_ids" bigint[], "worker_id" uuid) RETURNS SETOF "pgflow"."step_task_record" LANGUAGE sql SET "search_path" = '' AS $$
|
|
991
|
+
with tasks as (
|
|
992
|
+
select
|
|
993
|
+
task.flow_slug,
|
|
994
|
+
task.run_id,
|
|
995
|
+
task.step_slug,
|
|
996
|
+
task.task_index,
|
|
997
|
+
task.message_id
|
|
998
|
+
from pgflow.step_tasks as task
|
|
999
|
+
join pgflow.runs r on r.run_id = task.run_id
|
|
1000
|
+
where task.flow_slug = start_tasks.flow_slug
|
|
1001
|
+
and task.message_id = any(msg_ids)
|
|
1002
|
+
and task.status = 'queued'
|
|
1003
|
+
-- MVP: Don't start tasks on failed runs
|
|
1004
|
+
and r.status != 'failed'
|
|
1005
|
+
),
|
|
1006
|
+
start_tasks_update as (
|
|
1007
|
+
update pgflow.step_tasks
|
|
1008
|
+
set
|
|
1009
|
+
attempts_count = attempts_count + 1,
|
|
1010
|
+
status = 'started',
|
|
1011
|
+
started_at = now(),
|
|
1012
|
+
last_worker_id = worker_id
|
|
1013
|
+
from tasks
|
|
1014
|
+
where step_tasks.message_id = tasks.message_id
|
|
1015
|
+
and step_tasks.flow_slug = tasks.flow_slug
|
|
1016
|
+
and step_tasks.status = 'queued'
|
|
1017
|
+
),
|
|
1018
|
+
runs as (
|
|
1019
|
+
select
|
|
1020
|
+
r.run_id,
|
|
1021
|
+
r.input
|
|
1022
|
+
from pgflow.runs r
|
|
1023
|
+
where r.run_id in (select run_id from tasks)
|
|
1024
|
+
),
|
|
1025
|
+
deps as (
|
|
1026
|
+
select
|
|
1027
|
+
st.run_id,
|
|
1028
|
+
st.step_slug,
|
|
1029
|
+
dep.dep_slug,
|
|
1030
|
+
-- Aggregate map outputs or use single output
|
|
1031
|
+
CASE
|
|
1032
|
+
WHEN dep_step.step_type = 'map' THEN
|
|
1033
|
+
-- Aggregate all task outputs ordered by task_index
|
|
1034
|
+
-- Use COALESCE to return empty array if no tasks
|
|
1035
|
+
(SELECT COALESCE(jsonb_agg(dt.output ORDER BY dt.task_index), '[]'::jsonb)
|
|
1036
|
+
FROM pgflow.step_tasks dt
|
|
1037
|
+
WHERE dt.run_id = st.run_id
|
|
1038
|
+
AND dt.step_slug = dep.dep_slug
|
|
1039
|
+
AND dt.status = 'completed')
|
|
1040
|
+
ELSE
|
|
1041
|
+
-- Single step: use the single task output
|
|
1042
|
+
dep_task.output
|
|
1043
|
+
END as dep_output
|
|
1044
|
+
from tasks st
|
|
1045
|
+
join pgflow.deps dep on dep.flow_slug = st.flow_slug and dep.step_slug = st.step_slug
|
|
1046
|
+
join pgflow.steps dep_step on dep_step.flow_slug = dep.flow_slug and dep_step.step_slug = dep.dep_slug
|
|
1047
|
+
left join pgflow.step_tasks dep_task on
|
|
1048
|
+
dep_task.run_id = st.run_id and
|
|
1049
|
+
dep_task.step_slug = dep.dep_slug and
|
|
1050
|
+
dep_task.status = 'completed'
|
|
1051
|
+
and dep_step.step_type = 'single' -- Only join for single steps
|
|
1052
|
+
),
|
|
1053
|
+
deps_outputs as (
|
|
1054
|
+
select
|
|
1055
|
+
d.run_id,
|
|
1056
|
+
d.step_slug,
|
|
1057
|
+
jsonb_object_agg(d.dep_slug, d.dep_output) as deps_output,
|
|
1058
|
+
count(*) as dep_count
|
|
1059
|
+
from deps d
|
|
1060
|
+
group by d.run_id, d.step_slug
|
|
1061
|
+
),
|
|
1062
|
+
timeouts as (
|
|
1063
|
+
select
|
|
1064
|
+
task.message_id,
|
|
1065
|
+
task.flow_slug,
|
|
1066
|
+
coalesce(step.opt_timeout, flow.opt_timeout) + 2 as vt_delay
|
|
1067
|
+
from tasks task
|
|
1068
|
+
join pgflow.flows flow on flow.flow_slug = task.flow_slug
|
|
1069
|
+
join pgflow.steps step on step.flow_slug = task.flow_slug and step.step_slug = task.step_slug
|
|
1070
|
+
),
|
|
1071
|
+
-- Batch update visibility timeouts for all messages
|
|
1072
|
+
set_vt_batch as (
|
|
1073
|
+
select pgflow.set_vt_batch(
|
|
1074
|
+
start_tasks.flow_slug,
|
|
1075
|
+
array_agg(t.message_id order by t.message_id),
|
|
1076
|
+
array_agg(t.vt_delay order by t.message_id)
|
|
1077
|
+
)
|
|
1078
|
+
from timeouts t
|
|
1079
|
+
)
|
|
1080
|
+
select
|
|
1081
|
+
st.flow_slug,
|
|
1082
|
+
st.run_id,
|
|
1083
|
+
st.step_slug,
|
|
1084
|
+
-- ==========================================
|
|
1085
|
+
-- INPUT CONSTRUCTION LOGIC
|
|
1086
|
+
-- ==========================================
|
|
1087
|
+
-- This nested CASE statement determines how to construct the input
|
|
1088
|
+
-- for each task based on the step type (map vs non-map).
|
|
1089
|
+
--
|
|
1090
|
+
-- The fundamental difference:
|
|
1091
|
+
-- - Map steps: Receive RAW array elements (e.g., just 42 or "hello")
|
|
1092
|
+
-- - Non-map steps: Receive structured objects with named keys
|
|
1093
|
+
-- (e.g., {"run": {...}, "dependency1": {...}})
|
|
1094
|
+
-- ==========================================
|
|
1095
|
+
CASE
|
|
1096
|
+
-- -------------------- MAP STEPS --------------------
|
|
1097
|
+
-- Map steps process arrays element-by-element.
|
|
1098
|
+
-- Each task receives ONE element from the array at its task_index position.
|
|
1099
|
+
WHEN step.step_type = 'map' THEN
|
|
1100
|
+
-- Map steps get raw array elements without any wrapper object
|
|
1101
|
+
CASE
|
|
1102
|
+
-- ROOT MAP: Gets array from run input
|
|
1103
|
+
-- Example: run input = [1, 2, 3]
|
|
1104
|
+
-- task 0 gets: 1
|
|
1105
|
+
-- task 1 gets: 2
|
|
1106
|
+
-- task 2 gets: 3
|
|
1107
|
+
WHEN step.deps_count = 0 THEN
|
|
1108
|
+
-- Root map (deps_count = 0): no dependencies, reads from run input.
|
|
1109
|
+
-- Extract the element at task_index from the run's input array.
|
|
1110
|
+
-- Note: If run input is not an array, this will return NULL
|
|
1111
|
+
-- and the flow will fail (validated in start_flow).
|
|
1112
|
+
jsonb_array_element(r.input, st.task_index)
|
|
1113
|
+
|
|
1114
|
+
-- DEPENDENT MAP: Gets array from its single dependency
|
|
1115
|
+
-- Example: dependency output = ["a", "b", "c"]
|
|
1116
|
+
-- task 0 gets: "a"
|
|
1117
|
+
-- task 1 gets: "b"
|
|
1118
|
+
-- task 2 gets: "c"
|
|
1119
|
+
ELSE
|
|
1120
|
+
-- Has dependencies (should be exactly 1 for map steps).
|
|
1121
|
+
-- Extract the element at task_index from the dependency's output array.
|
|
1122
|
+
--
|
|
1123
|
+
-- Why the subquery with jsonb_each?
|
|
1124
|
+
-- - The dependency outputs a raw array: [1, 2, 3]
|
|
1125
|
+
-- - deps_outputs aggregates it into: {"dep_name": [1, 2, 3]}
|
|
1126
|
+
-- - We need to unwrap and get just the array value
|
|
1127
|
+
-- - Map steps have exactly 1 dependency (enforced by add_step)
|
|
1128
|
+
-- - So jsonb_each will return exactly 1 row
|
|
1129
|
+
-- - We extract the 'value' which is the raw array [1, 2, 3]
|
|
1130
|
+
-- - Then get the element at task_index from that array
|
|
1131
|
+
(SELECT jsonb_array_element(value, st.task_index)
|
|
1132
|
+
FROM jsonb_each(dep_out.deps_output)
|
|
1133
|
+
LIMIT 1)
|
|
1134
|
+
END
|
|
1135
|
+
|
|
1136
|
+
-- -------------------- NON-MAP STEPS --------------------
|
|
1137
|
+
-- Regular (non-map) steps receive ALL inputs as a structured object.
|
|
1138
|
+
-- This includes the original run input plus all dependency outputs.
|
|
1139
|
+
ELSE
|
|
1140
|
+
-- Non-map steps get structured input with named keys
|
|
1141
|
+
-- Example output: {
|
|
1142
|
+
-- "run": {"original": "input"},
|
|
1143
|
+
-- "step1": {"output": "from_step1"},
|
|
1144
|
+
-- "step2": {"output": "from_step2"}
|
|
1145
|
+
-- }
|
|
1146
|
+
--
|
|
1147
|
+
-- Build object with 'run' key containing original input
|
|
1148
|
+
jsonb_build_object('run', r.input) ||
|
|
1149
|
+
-- Merge with deps_output which already has dependency outputs
|
|
1150
|
+
-- deps_output format: {"dep1": output1, "dep2": output2, ...}
|
|
1151
|
+
-- If no dependencies, defaults to empty object
|
|
1152
|
+
coalesce(dep_out.deps_output, '{}'::jsonb)
|
|
1153
|
+
END as input,
|
|
1154
|
+
st.message_id as msg_id,
|
|
1155
|
+
st.task_index as task_index
|
|
1156
|
+
from tasks st
|
|
1157
|
+
join runs r on st.run_id = r.run_id
|
|
1158
|
+
join pgflow.steps step on
|
|
1159
|
+
step.flow_slug = st.flow_slug and
|
|
1160
|
+
step.step_slug = st.step_slug
|
|
1161
|
+
left join deps_outputs dep_out on
|
|
1162
|
+
dep_out.run_id = st.run_id and
|
|
1163
|
+
dep_out.step_slug = st.step_slug
|
|
1164
|
+
$$;
|
|
1165
|
+
-- Create "add_step" function
|
|
1166
|
+
CREATE FUNCTION "pgflow"."add_step" ("flow_slug" text, "step_slug" text, "deps_slugs" text[] DEFAULT '{}', "max_attempts" integer DEFAULT NULL::integer, "base_delay" integer DEFAULT NULL::integer, "timeout" integer DEFAULT NULL::integer, "start_delay" integer DEFAULT NULL::integer, "step_type" text DEFAULT 'single') RETURNS "pgflow"."steps" LANGUAGE plpgsql SET "search_path" = '' AS $$
|
|
1167
|
+
DECLARE
|
|
1168
|
+
result_step pgflow.steps;
|
|
1169
|
+
next_idx int;
|
|
1170
|
+
BEGIN
|
|
1171
|
+
-- Validate map step constraints
|
|
1172
|
+
-- Map steps can have either:
|
|
1173
|
+
-- 0 dependencies (root map - maps over flow input array)
|
|
1174
|
+
-- 1 dependency (dependent map - maps over dependency output array)
|
|
1175
|
+
IF COALESCE(add_step.step_type, 'single') = 'map' AND COALESCE(array_length(add_step.deps_slugs, 1), 0) > 1 THEN
|
|
1176
|
+
RAISE EXCEPTION 'Map step "%" can have at most one dependency, but % were provided: %',
|
|
1177
|
+
add_step.step_slug,
|
|
1178
|
+
COALESCE(array_length(add_step.deps_slugs, 1), 0),
|
|
1179
|
+
array_to_string(add_step.deps_slugs, ', ');
|
|
1180
|
+
END IF;
|
|
1181
|
+
|
|
1182
|
+
-- Get next step index
|
|
1183
|
+
SELECT COALESCE(MAX(s.step_index) + 1, 0) INTO next_idx
|
|
1184
|
+
FROM pgflow.steps s
|
|
1185
|
+
WHERE s.flow_slug = add_step.flow_slug;
|
|
1186
|
+
|
|
1187
|
+
-- Create the step
|
|
1188
|
+
INSERT INTO pgflow.steps (
|
|
1189
|
+
flow_slug, step_slug, step_type, step_index, deps_count,
|
|
1190
|
+
opt_max_attempts, opt_base_delay, opt_timeout, opt_start_delay
|
|
1191
|
+
)
|
|
1192
|
+
VALUES (
|
|
1193
|
+
add_step.flow_slug,
|
|
1194
|
+
add_step.step_slug,
|
|
1195
|
+
COALESCE(add_step.step_type, 'single'),
|
|
1196
|
+
next_idx,
|
|
1197
|
+
COALESCE(array_length(add_step.deps_slugs, 1), 0),
|
|
1198
|
+
add_step.max_attempts,
|
|
1199
|
+
add_step.base_delay,
|
|
1200
|
+
add_step.timeout,
|
|
1201
|
+
add_step.start_delay
|
|
1202
|
+
)
|
|
1203
|
+
ON CONFLICT ON CONSTRAINT steps_pkey
|
|
1204
|
+
DO UPDATE SET step_slug = EXCLUDED.step_slug
|
|
1205
|
+
RETURNING * INTO result_step;
|
|
1206
|
+
|
|
1207
|
+
-- Insert dependencies
|
|
1208
|
+
INSERT INTO pgflow.deps (flow_slug, dep_slug, step_slug)
|
|
1209
|
+
SELECT add_step.flow_slug, d.dep_slug, add_step.step_slug
|
|
1210
|
+
FROM unnest(COALESCE(add_step.deps_slugs, '{}')) AS d(dep_slug)
|
|
1211
|
+
WHERE add_step.deps_slugs IS NOT NULL AND array_length(add_step.deps_slugs, 1) > 0
|
|
1212
|
+
ON CONFLICT ON CONSTRAINT deps_pkey DO NOTHING;
|
|
1213
|
+
|
|
1214
|
+
RETURN result_step;
|
|
1215
|
+
END;
|
|
1216
|
+
$$;
|
|
1217
|
+
-- Drop "add_step" function
|
|
1218
|
+
DROP FUNCTION "pgflow"."add_step" (text, text, integer, integer, integer, integer);
|
|
1219
|
+
-- Drop "add_step" function
|
|
1220
|
+
DROP FUNCTION "pgflow"."add_step" (text, text, text[], integer, integer, integer, integer);
|
package/dist/types.d.ts
CHANGED
|
@@ -24,7 +24,7 @@ export type StepTaskRecord<TFlow extends AnyFlow> = {
|
|
|
24
24
|
* Composite key that is enough to find a particular step task
|
|
25
25
|
* Contains only the minimum fields needed to identify a task
|
|
26
26
|
*/
|
|
27
|
-
export type StepTaskKey = Pick<StepTaskRecord<
|
|
27
|
+
export type StepTaskKey = Pick<StepTaskRecord<AnyFlow>, 'run_id' | 'step_slug' | 'task_index'>;
|
|
28
28
|
/**
|
|
29
29
|
* Record representing a message from queue polling
|
|
30
30
|
*/
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,OAAO,EACP,gBAAgB,EACjB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,MAAM,IAAI,GACZ,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAAE,GACnC,IAAI,EAAE,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,KAAK,SAAS,OAAO,IAAI;KACjD,QAAQ,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,GAAG;QAC5D,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,QAAQ,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5C,MAAM,EAAE,MAAM,CAAC;KAChB;CACF,CAAC,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAElD;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,OAAO,EACP,gBAAgB,EACjB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,MAAM,IAAI,GACZ,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAAE,GACnC,IAAI,EAAE,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,KAAK,SAAS,OAAO,IAAI;KACjD,QAAQ,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,GAAG;QAC5D,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,QAAQ,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5C,MAAM,EAAE,MAAM,CAAC;KAChB;CACF,CAAC,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAElD;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,YAAY,CAAC,CAAC;AAI/F;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,IAAI,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO;IAC5D;;OAEG;IACH,SAAS,CAAC,KAAK,SAAS,OAAO,EAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAC9B,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB;;;;;;;OAOG;IACH,YAAY,CACV,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,EACzB,SAAS,EAAE,MAAM,EACjB,cAAc,CAAC,EAAE,MAAM,EACvB,cAAc,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IAE5B;;;;;OAKG;IACH,UAAU,CACR,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAEpC;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElE;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;AAEjE;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;AAEjE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC;AAE9E;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pgflow/core",
|
|
3
|
-
"version": "0.0.0-array-map-steps-
|
|
3
|
+
"version": "0.0.0-array-map-steps-b956f8f9-20251006084236",
|
|
4
4
|
"license": "AGPL-3.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"postgres": "^3.4.5",
|
|
27
|
-
"@pgflow/dsl": "0.0.0-array-map-steps-
|
|
27
|
+
"@pgflow/dsl": "0.0.0-array-map-steps-b956f8f9-20251006084236"
|
|
28
28
|
},
|
|
29
29
|
"publishConfig": {
|
|
30
30
|
"access": "public"
|