@pgflow/core 0.0.0-add-workerconfig-to-context--20250905094004-b98e1fec-20250905074005 → 0.0.0-array-map-steps-302d00a8-20250925065142

package/dist/supabase/migrations/20250912075001_pgflow_temp_pr1_schema.sql

@@ -0,0 +1,185 @@
+-- Modify "step_states" table
+ALTER TABLE "pgflow"."step_states" DROP CONSTRAINT "step_states_remaining_tasks_check", ADD CONSTRAINT "remaining_tasks_state_consistency" CHECK ((remaining_tasks IS NULL) OR (status <> 'created'::text)), ADD CONSTRAINT "step_states_initial_tasks_check" CHECK (initial_tasks >= 0), ALTER COLUMN "remaining_tasks" DROP NOT NULL, ALTER COLUMN "remaining_tasks" DROP DEFAULT, ADD COLUMN "initial_tasks" integer NULL DEFAULT 1;
+-- Modify "step_tasks" table
+ALTER TABLE "pgflow"."step_tasks" DROP CONSTRAINT "only_single_task_per_step";
+-- Modify "steps" table
+ALTER TABLE "pgflow"."steps" DROP CONSTRAINT "steps_step_type_check", ADD CONSTRAINT "steps_step_type_check" CHECK (step_type = ANY (ARRAY['single'::text, 'map'::text]));
+-- Modify "start_ready_steps" function
+CREATE OR REPLACE FUNCTION "pgflow"."start_ready_steps" ("run_id" uuid) RETURNS void LANGUAGE sql SET "search_path" = '' AS $$
+WITH ready_steps AS (
+  SELECT *
+  FROM pgflow.step_states AS step_state
+  WHERE step_state.run_id = start_ready_steps.run_id
+    AND step_state.status = 'created'
+    AND step_state.remaining_deps = 0
+  ORDER BY step_state.step_slug
+  FOR UPDATE
+),
+started_step_states AS (
+  UPDATE pgflow.step_states
+  SET status = 'started',
+      started_at = now(),
+      remaining_tasks = ready_steps.initial_tasks  -- Copy initial_tasks to remaining_tasks when starting
+  FROM ready_steps
+  WHERE pgflow.step_states.run_id = start_ready_steps.run_id
+    AND pgflow.step_states.step_slug = ready_steps.step_slug
+  RETURNING pgflow.step_states.*
+),
+sent_messages AS (
+  SELECT
+    started_step.flow_slug,
+    started_step.run_id,
+    started_step.step_slug,
+    pgmq.send(
+      started_step.flow_slug, 
+      jsonb_build_object(
+        'flow_slug', started_step.flow_slug,
+        'run_id', started_step.run_id,
+        'step_slug', started_step.step_slug,
+        'task_index', 0
+      ),
+      COALESCE(step.opt_start_delay, 0)
+    ) AS msg_id
+  FROM started_step_states AS started_step
+  JOIN pgflow.steps AS step 
+    ON step.flow_slug = started_step.flow_slug 
+    AND step.step_slug = started_step.step_slug
+),
+broadcast_events AS (
+  SELECT 
+    realtime.send(
+      jsonb_build_object(
+        'event_type', 'step:started',
+        'run_id', started_step.run_id,
+        'step_slug', started_step.step_slug,
+        'status', 'started',
+        'started_at', started_step.started_at,
+        'remaining_tasks', started_step.remaining_tasks,
+        'remaining_deps', started_step.remaining_deps
+      ),
+      concat('step:', started_step.step_slug, ':started'),
+      concat('pgflow:run:', started_step.run_id),
+      false
+    )
+  FROM started_step_states AS started_step
+)
+INSERT INTO pgflow.step_tasks (flow_slug, run_id, step_slug, message_id)
+SELECT
+  sent_messages.flow_slug,
+  sent_messages.run_id,
+  sent_messages.step_slug,
+  sent_messages.msg_id
+FROM sent_messages;
+$$;
+-- Modify "start_flow" function
+CREATE OR REPLACE FUNCTION "pgflow"."start_flow" ("flow_slug" text, "input" jsonb, "run_id" uuid DEFAULT NULL::uuid) RETURNS SETOF "pgflow"."runs" LANGUAGE plpgsql SET "search_path" = '' AS $$
+declare
+  v_created_run pgflow.runs%ROWTYPE;
+begin
+
+WITH
+  flow_steps AS (
+    SELECT steps.flow_slug, steps.step_slug, steps.deps_count
+    FROM pgflow.steps
+    WHERE steps.flow_slug = start_flow.flow_slug
+  ),
+  created_run AS (
+    INSERT INTO pgflow.runs (run_id, flow_slug, input, remaining_steps)
+    VALUES (
+      COALESCE(start_flow.run_id, gen_random_uuid()),
+      start_flow.flow_slug,
+      start_flow.input,
+      (SELECT count(*) FROM flow_steps)
+    )
+    RETURNING *
+  ),
+  created_step_states AS (
+    INSERT INTO pgflow.step_states (flow_slug, run_id, step_slug, remaining_deps, initial_tasks)
+    SELECT
+      fs.flow_slug,
+      (SELECT created_run.run_id FROM created_run),
+      fs.step_slug,
+      fs.deps_count,
+      1  -- For now, all steps get initial_tasks = 1 (single steps)
+    FROM flow_steps fs
+  )
+SELECT * FROM created_run INTO v_created_run;
+
+-- Send broadcast event for run started
+PERFORM realtime.send(
+  jsonb_build_object(
+    'event_type', 'run:started',
+    'run_id', v_created_run.run_id,
+    'flow_slug', v_created_run.flow_slug,
+    'input', v_created_run.input,
+    'status', 'started',
+    'remaining_steps', v_created_run.remaining_steps,
+    'started_at', v_created_run.started_at
+  ),
+  'run:started',
+  concat('pgflow:run:', v_created_run.run_id),
+  false
+);
+
+PERFORM pgflow.start_ready_steps(v_created_run.run_id);
+
+RETURN QUERY SELECT * FROM pgflow.runs where pgflow.runs.run_id = v_created_run.run_id;
+
+end;
+$$;
+-- Create "add_step" function
+CREATE FUNCTION "pgflow"."add_step" ("flow_slug" text, "step_slug" text, "deps_slugs" text[] DEFAULT '{}', "max_attempts" integer DEFAULT NULL::integer, "base_delay" integer DEFAULT NULL::integer, "timeout" integer DEFAULT NULL::integer, "start_delay" integer DEFAULT NULL::integer, "step_type" text DEFAULT 'single') RETURNS "pgflow"."steps" LANGUAGE plpgsql SET "search_path" = '' AS $$
+DECLARE
+  result_step pgflow.steps;
+  next_idx int;
+BEGIN
+  -- Validate map step constraints
+  -- Map steps can have either:
+  --   0 dependencies (root map - maps over flow input array)
+  --   1 dependency (dependent map - maps over dependency output array)
+  IF COALESCE(add_step.step_type, 'single') = 'map' AND COALESCE(array_length(add_step.deps_slugs, 1), 0) > 1 THEN
+    RAISE EXCEPTION 'Map step "%" can have at most one dependency, but % were provided: %', 
+      add_step.step_slug,
+      COALESCE(array_length(add_step.deps_slugs, 1), 0),
+      array_to_string(add_step.deps_slugs, ', ');
+  END IF;
+
+  -- Get next step index
+  SELECT COALESCE(MAX(s.step_index) + 1, 0) INTO next_idx
+  FROM pgflow.steps s
+  WHERE s.flow_slug = add_step.flow_slug;
+
+  -- Create the step
+  INSERT INTO pgflow.steps (
+    flow_slug, step_slug, step_type, step_index, deps_count,
+    opt_max_attempts, opt_base_delay, opt_timeout, opt_start_delay
+  )
+  VALUES (
+    add_step.flow_slug,
+    add_step.step_slug,
+    COALESCE(add_step.step_type, 'single'),
+    next_idx, 
+    COALESCE(array_length(add_step.deps_slugs, 1), 0),
+    add_step.max_attempts,
+    add_step.base_delay,
+    add_step.timeout,
+    add_step.start_delay
+  )
+  ON CONFLICT ON CONSTRAINT steps_pkey
+  DO UPDATE SET step_slug = EXCLUDED.step_slug
+  RETURNING * INTO result_step;
+
+  -- Insert dependencies
+  INSERT INTO pgflow.deps (flow_slug, dep_slug, step_slug)
+  SELECT add_step.flow_slug, d.dep_slug, add_step.step_slug
+  FROM unnest(COALESCE(add_step.deps_slugs, '{}')) AS d(dep_slug)
+  WHERE add_step.deps_slugs IS NOT NULL AND array_length(add_step.deps_slugs, 1) > 0
+  ON CONFLICT ON CONSTRAINT deps_pkey DO NOTHING;
+  
+  RETURN result_step;
+END;
+$$;
+-- Drop "add_step" function
+DROP FUNCTION "pgflow"."add_step" (text, text, integer, integer, integer, integer);
+-- Drop "add_step" function
+DROP FUNCTION "pgflow"."add_step" (text, text, text[], integer, integer, integer, integer);

package/dist/supabase/migrations/20250912080800_pgflow_temp_pr2_root_maps.sql

@@ -0,0 +1,95 @@
+-- Modify "step_states" table
+ALTER TABLE "pgflow"."step_states" ALTER COLUMN "initial_tasks" SET NOT NULL;
+-- Modify "start_flow" function
+CREATE OR REPLACE FUNCTION "pgflow"."start_flow" ("flow_slug" text, "input" jsonb, "run_id" uuid DEFAULT NULL::uuid) RETURNS SETOF "pgflow"."runs" LANGUAGE plpgsql SET "search_path" = '' AS $$
+declare
+  v_created_run pgflow.runs%ROWTYPE;
+  v_root_map_count int;
+begin
+
+-- Check for root map steps and validate input
+WITH root_maps AS (
+  SELECT step_slug
+  FROM pgflow.steps
+  WHERE steps.flow_slug = start_flow.flow_slug
+    AND steps.step_type = 'map'
+    AND steps.deps_count = 0
+)
+SELECT COUNT(*) INTO v_root_map_count FROM root_maps;
+
+-- If we have root map steps, validate that input is an array
+IF v_root_map_count > 0 THEN
+  -- First check for NULL (should be caught by NOT NULL constraint, but be defensive)
+  IF start_flow.input IS NULL THEN
+    RAISE EXCEPTION 'Flow % has root map steps but input is NULL', start_flow.flow_slug;
+  END IF;
+  
+  -- Then check if it's not an array
+  IF jsonb_typeof(start_flow.input) != 'array' THEN
+    RAISE EXCEPTION 'Flow % has root map steps but input is not an array (got %)', 
+      start_flow.flow_slug, jsonb_typeof(start_flow.input);
+  END IF;
+END IF;
+
+WITH
+  flow_steps AS (
+    SELECT steps.flow_slug, steps.step_slug, steps.step_type, steps.deps_count
+    FROM pgflow.steps
+    WHERE steps.flow_slug = start_flow.flow_slug
+  ),
+  created_run AS (
+    INSERT INTO pgflow.runs (run_id, flow_slug, input, remaining_steps)
+    VALUES (
+      COALESCE(start_flow.run_id, gen_random_uuid()),
+      start_flow.flow_slug,
+      start_flow.input,
+      (SELECT count(*) FROM flow_steps)
+    )
+    RETURNING *
+  ),
+  created_step_states AS (
+    INSERT INTO pgflow.step_states (flow_slug, run_id, step_slug, remaining_deps, initial_tasks)
+    SELECT
+      fs.flow_slug,
+      (SELECT created_run.run_id FROM created_run),
+      fs.step_slug,
+      fs.deps_count,
+      -- For root map steps (map with no deps), set initial_tasks to array length
+      -- For all other steps, set initial_tasks to 1
+      CASE 
+        WHEN fs.step_type = 'map' AND fs.deps_count = 0 THEN 
+          CASE 
+            WHEN jsonb_typeof(start_flow.input) = 'array' THEN 
+              jsonb_array_length(start_flow.input)
+            ELSE 
+              1
+          END
+        ELSE 
+          1
+      END
+    FROM flow_steps fs
+  )
+SELECT * FROM created_run INTO v_created_run;
+
+-- Send broadcast event for run started
+PERFORM realtime.send(
+  jsonb_build_object(
+    'event_type', 'run:started',
+    'run_id', v_created_run.run_id,
+    'flow_slug', v_created_run.flow_slug,
+    'input', v_created_run.input,
+    'status', 'started',
+    'remaining_steps', v_created_run.remaining_steps,
+    'started_at', v_created_run.started_at
+  ),
+  'run:started',
+  concat('pgflow:run:', v_created_run.run_id),
+  false
+);
+
+PERFORM pgflow.start_ready_steps(v_created_run.run_id);
+
+RETURN QUERY SELECT * FROM pgflow.runs where pgflow.runs.run_id = v_created_run.run_id;
+
+end;
+$$;

package/dist/supabase/migrations/20250912125339_pgflow_TEMP_task_spawning_optimization.sql

@@ -0,0 +1,146 @@
+-- Modify "start_ready_steps" function
+CREATE OR REPLACE FUNCTION "pgflow"."start_ready_steps" ("run_id" uuid) RETURNS void LANGUAGE sql SET "search_path" = '' AS $$
+-- First handle empty array map steps (initial_tasks = 0) - direct transition to completed
+WITH empty_map_steps AS (
+  SELECT step_state.*
+  FROM pgflow.step_states AS step_state
+  JOIN pgflow.steps AS step 
+    ON step.flow_slug = step_state.flow_slug 
+    AND step.step_slug = step_state.step_slug
+  WHERE step_state.run_id = start_ready_steps.run_id
+    AND step_state.status = 'created'
+    AND step_state.remaining_deps = 0
+    AND step.step_type = 'map'
+    AND step_state.initial_tasks = 0
+  ORDER BY step_state.step_slug
+  FOR UPDATE OF step_state
+),
+completed_empty_steps AS (
+  UPDATE pgflow.step_states
+  SET status = 'completed',
+      started_at = now(),
+      completed_at = now(),
+      remaining_tasks = 0
+  FROM empty_map_steps
+  WHERE pgflow.step_states.run_id = start_ready_steps.run_id
+    AND pgflow.step_states.step_slug = empty_map_steps.step_slug
+  RETURNING pgflow.step_states.*
+),
+broadcast_empty_completed AS (
+  SELECT 
+    realtime.send(
+      jsonb_build_object(
+        'event_type', 'step:completed',
+        'run_id', completed_step.run_id,
+        'step_slug', completed_step.step_slug,
+        'status', 'completed',
+        'started_at', completed_step.started_at,
+        'completed_at', completed_step.completed_at,
+        'remaining_tasks', 0,
+        'remaining_deps', 0,
+        'output', '[]'::jsonb
+      ),
+      concat('step:', completed_step.step_slug, ':completed'),
+      concat('pgflow:run:', completed_step.run_id),
+      false
+    )
+  FROM completed_empty_steps AS completed_step
+),
+
+-- Now handle non-empty steps (both single and map with initial_tasks > 0)
+ready_steps AS (
+  SELECT *
+  FROM pgflow.step_states AS step_state
+  WHERE step_state.run_id = start_ready_steps.run_id
+    AND step_state.status = 'created'
+    AND step_state.remaining_deps = 0
+    -- Exclude empty map steps already handled
+    AND NOT EXISTS (
+      SELECT 1 FROM empty_map_steps 
+      WHERE empty_map_steps.run_id = step_state.run_id 
+        AND empty_map_steps.step_slug = step_state.step_slug
+    )
+  ORDER BY step_state.step_slug
+  FOR UPDATE
+),
+started_step_states AS (
+  UPDATE pgflow.step_states
+  SET status = 'started',
+      started_at = now(),
+      remaining_tasks = ready_steps.initial_tasks  -- Copy initial_tasks to remaining_tasks when starting
+  FROM ready_steps
+  WHERE pgflow.step_states.run_id = start_ready_steps.run_id
+    AND pgflow.step_states.step_slug = ready_steps.step_slug
+  RETURNING pgflow.step_states.*
+),
+
+-- Generate tasks based on initial_tasks count
+-- For single steps: initial_tasks = 1, so generate_series(0, 0) = single task with index 0
+-- For map steps: initial_tasks = N, so generate_series(0, N-1) = N tasks with indices 0..N-1
+-- Group messages by step for batch sending
+message_batches AS (
+  SELECT
+    started_step.flow_slug,
+    started_step.run_id,
+    started_step.step_slug,
+    COALESCE(step.opt_start_delay, 0) as delay,
+    array_agg(
+      jsonb_build_object(
+        'flow_slug', started_step.flow_slug,
+        'run_id', started_step.run_id,
+        'step_slug', started_step.step_slug,
+        'task_index', task_idx.task_index
+      ) ORDER BY task_idx.task_index
+    ) AS messages,
+    array_agg(task_idx.task_index ORDER BY task_idx.task_index) AS task_indices
+  FROM started_step_states AS started_step
+  JOIN pgflow.steps AS step 
+    ON step.flow_slug = started_step.flow_slug 
+    AND step.step_slug = started_step.step_slug
+  -- Generate task indices from 0 to initial_tasks-1
+  CROSS JOIN LATERAL generate_series(0, started_step.initial_tasks - 1) AS task_idx(task_index)
+  GROUP BY started_step.flow_slug, started_step.run_id, started_step.step_slug, step.opt_start_delay
+),
+-- Send messages in batch for better performance with large arrays
+sent_messages AS (
+  SELECT
+    mb.flow_slug,
+    mb.run_id,
+    mb.step_slug,
+    task_indices.task_index,
+    msg_ids.msg_id
+  FROM message_batches mb
+  CROSS JOIN LATERAL unnest(mb.task_indices) WITH ORDINALITY AS task_indices(task_index, idx_ord)
+  CROSS JOIN LATERAL pgmq.send_batch(mb.flow_slug, mb.messages, mb.delay) WITH ORDINALITY AS msg_ids(msg_id, msg_ord)
+  WHERE task_indices.idx_ord = msg_ids.msg_ord
+),
+
+broadcast_events AS (
+  SELECT 
+    realtime.send(
+      jsonb_build_object(
+        'event_type', 'step:started',
+        'run_id', started_step.run_id,
+        'step_slug', started_step.step_slug,
+        'status', 'started',
+        'started_at', started_step.started_at,
+        'remaining_tasks', started_step.remaining_tasks,
+        'remaining_deps', started_step.remaining_deps
+      ),
+      concat('step:', started_step.step_slug, ':started'),
+      concat('pgflow:run:', started_step.run_id),
+      false
+    )
+  FROM started_step_states AS started_step
+)
+
+-- Insert all generated tasks with their respective task_index values
+INSERT INTO pgflow.step_tasks (flow_slug, run_id, step_slug, task_index, message_id)
+SELECT
+  sent_messages.flow_slug,
+  sent_messages.run_id,
+  sent_messages.step_slug,
+  sent_messages.task_index,
+  sent_messages.msg_id
+FROM sent_messages;
+$$;