@pgflow/core 0.1.18 → 0.1.20

package/dist/supabase/migrations/000000_schema.sql

@@ -1,149 +0,0 @@
-create extension if not exists pgmq version '1.4.4';
-
-create schema if not exists pgflow;
-
---------------------------------------------------------------------------
------------------- TODO: fix me, UNSECURE --------------------------------
---------------------------------------------------------------------------
-grant usage on schema pgflow to anon, authenticated, service_role;
-grant all on all tables in schema pgflow to anon, authenticated, service_role;
-grant all on all routines in schema pgflow to anon, authenticated, service_role;
-grant all on all sequences in schema pgflow to anon,
-authenticated,
-service_role;
-alter default privileges for role postgres in schema pgflow
-grant all on tables to anon, authenticated, service_role;
-alter default privileges for role postgres in schema pgflow
-grant all on routines to anon, authenticated, service_role;
-alter default privileges for role postgres in schema pgflow
-grant all on sequences to anon, authenticated, service_role;
-
-------------------------------------------
--- Core flow definition tables
-------------------------------------------
-
------ check constraint helper function -------
-create or replace function pgflow.is_valid_slug(
-  slug text
-)
-returns boolean
-language plpgsql
-immutable
-as $$
-begin
-    return
-      slug is not null
-      and slug <> ''
-      and length(slug) <= 128
-      and slug ~ '^[a-zA-Z_][a-zA-Z0-9_]*$'
-      and slug NOT IN ('run'); -- reserved words
-end;
-$$;
-
--- Flows table - stores flow definitions
-create table pgflow.flows (
-  flow_slug text primary key not null,  -- Unique identifier for the flow
-  opt_max_attempts int not null default 3,
-  opt_base_delay int not null default 1,
-  opt_timeout int not null default 60,
-  constraint slug_is_valid check (pgflow.is_valid_slug(flow_slug)),
-  constraint opt_max_attempts_is_nonnegative check (opt_max_attempts >= 0),
-  constraint opt_base_delay_is_nonnegative check (opt_base_delay >= 0),
-  constraint opt_timeout_is_positive check (opt_timeout > 0)
-);
-
--- Steps table - stores individual steps within flows
-create table pgflow.steps (
-  flow_slug text not null references pgflow.flows (flow_slug),
-  step_slug text not null,
-  step_type text not null default 'single',
-  deps_count int not null default 0 check (deps_count >= 0),
-  opt_max_attempts int,
-  opt_base_delay int,
-  opt_timeout int,
-  primary key (flow_slug, step_slug),
-  check (pgflow.is_valid_slug(step_slug)),
-  check (step_type in ('single')),
-  constraint opt_max_attempts_is_nonnegative check (opt_max_attempts is null or opt_max_attempts >= 0),
-  constraint opt_base_delay_is_nonnegative check (opt_base_delay is null or opt_base_delay >= 0),
-  constraint opt_timeout_is_positive check (opt_timeout is null or opt_timeout > 0)
-);
-
--- Dependencies table - stores relationships between steps
-create table pgflow.deps (
-  flow_slug text not null references pgflow.flows (flow_slug),
-  dep_slug text not null, -- slug of the dependency
-  step_slug text not null, -- slug of the dependent
-  primary key (flow_slug, dep_slug, step_slug),
-  foreign key (flow_slug, dep_slug)
-  references pgflow.steps (flow_slug, step_slug),
-  foreign key (flow_slug, step_slug)
-  references pgflow.steps (flow_slug, step_slug),
-  check (dep_slug != step_slug)  -- Prevent self-dependencies
-);
-
-------------------------------------------
--- Runtime State Tables
-------------------------------------------
-
--- Runs table - tracks flow execution instances
-create table pgflow.runs (
-  run_id uuid primary key not null default gen_random_uuid(),
-  flow_slug text not null references pgflow.flows (flow_slug), -- denormalized
-  status text not null default 'started',
-  input jsonb not null,
-  output jsonb,
-  remaining_steps int not null default 0 check (remaining_steps >= 0),
-  check (status in ('started', 'failed', 'completed'))
-);
-
--- Step states table - tracks the state of individual steps within a run
-create table pgflow.step_states (
-  flow_slug text not null references pgflow.flows (flow_slug),
-  run_id uuid not null references pgflow.runs (run_id),
-  step_slug text not null,
-  status text not null default 'created',
-  remaining_tasks int not null default 1 check (remaining_tasks >= 0),
-  remaining_deps int not null default 0 check (remaining_deps >= 0),
-  primary key (run_id, step_slug),
-  foreign key (flow_slug, step_slug)
-  references pgflow.steps (flow_slug, step_slug),
-  check (status in ('created', 'started', 'completed', 'failed')),
-  check (status != 'completed' or remaining_tasks = 0)
-);
-
--- Step tasks table - tracks units of work for step
-create table pgflow.step_tasks (
-  flow_slug text not null references pgflow.flows (flow_slug),
-  run_id uuid not null references pgflow.runs (run_id),
-  step_slug text not null,
-  message_id bigint,
-  task_index int not null default 0,
-  status text not null default 'queued',
-  attempts_count int not null default 0,
-  error_message text,
-  output jsonb,
-  constraint step_tasks_pkey primary key (run_id, step_slug, task_index),
-  foreign key (run_id, step_slug)
-  references pgflow.step_states (run_id, step_slug),
-  constraint valid_status check (
-    status in ('queued', 'completed', 'failed')
-  ),
-  constraint output_valid_only_for_completed check (
-    output is null or status = 'completed'
-  ),
-  constraint only_single_task_per_step check (task_index = 0),
-  constraint attempts_count_nonnegative check (attempts_count >= 0)
-);
-
-------------------------------------------
--- Types
-------------------------------------------
-
-create type pgflow.step_task_record as (
-  flow_slug text,
-  run_id uuid,
-  step_slug text,
-  input jsonb,
-  msg_id bigint
-);

package/dist/supabase/migrations/000005_create_flow.sql

@@ -1,29 +0,0 @@
-create or replace function pgflow.create_flow(
-  flow_slug text,
-  max_attempts int default 3,
-  base_delay int default 5,
-  timeout int default 60
-)
-returns pgflow.flows
-language sql
-set search_path to ''
-volatile
-as $$
-WITH
-  flow_upsert AS (
-    INSERT INTO pgflow.flows (flow_slug, opt_max_attempts, opt_base_delay, opt_timeout)
-    VALUES (flow_slug, max_attempts, base_delay, timeout)
-    ON CONFLICT (flow_slug) DO UPDATE
-    SET flow_slug = pgflow.flows.flow_slug -- Dummy update
-    RETURNING *
-  ),
-  ensure_queue AS (
-    SELECT pgmq.create(flow_slug)
-    WHERE NOT EXISTS (
-      SELECT 1 FROM pgmq.list_queues() WHERE queue_name = flow_slug
-    )
-  )
-SELECT f.*
-FROM flow_upsert f
-LEFT JOIN (SELECT 1 FROM ensure_queue) _dummy ON true; -- Left join ensures flow is returned
-$$;

package/dist/supabase/migrations/000010_add_step.sql

@@ -1,48 +0,0 @@
-create or replace function pgflow.add_step(
-  flow_slug text,
-  step_slug text,
-  deps_slugs text [],
-  max_attempts int default null,
-  base_delay int default null,
-  timeout int default null
-)
-returns pgflow.steps
-language sql
-set search_path to ''
-volatile
-as $$
-WITH
-  create_step AS (
-    INSERT INTO pgflow.steps (flow_slug, step_slug, deps_count, opt_max_attempts, opt_base_delay, opt_timeout)
-    VALUES (flow_slug, step_slug, COALESCE(array_length(deps_slugs, 1), 0), max_attempts, base_delay, timeout)
-    ON CONFLICT (flow_slug, step_slug)
-    DO UPDATE SET step_slug = pgflow.steps.step_slug
-    RETURNING *
-  ),
-  insert_deps AS (
-    INSERT INTO pgflow.deps (flow_slug, dep_slug, step_slug)
-    SELECT add_step.flow_slug, d.dep_slug, add_step.step_slug
-    FROM unnest(deps_slugs) AS d(dep_slug)
-    ON CONFLICT (flow_slug, dep_slug, step_slug) DO NOTHING
-    RETURNING 1
-  )
--- Return the created step
-SELECT * FROM create_step;
-$$;
-
--- New overloaded function without deps_slugs parameter
-create or replace function pgflow.add_step(
-  flow_slug text,
-  step_slug text,
-  max_attempts int default null,
-  base_delay int default null,
-  timeout int default null
-)
-returns pgflow.steps
-language sql
-set search_path to ''
-volatile
-as $$
-    -- Call the original function with an empty array
-    SELECT * FROM pgflow.add_step(flow_slug, step_slug, ARRAY[]::text[], max_attempts, base_delay, timeout);
-$$;

package/dist/supabase/migrations/000015_start_ready_steps.sql

@@ -1,45 +0,0 @@
-create or replace function pgflow.start_ready_steps(run_id uuid)
-returns void
-language sql
-set search_path to ''
-as $$
-
-WITH ready_steps AS (
-  SELECT *
-  FROM pgflow.step_states AS step_state
-  WHERE step_state.run_id = start_ready_steps.run_id
-    AND step_state.status = 'created'
-    AND step_state.remaining_deps = 0
-  ORDER BY step_state.step_slug
-  FOR UPDATE
-),
-started_step_states AS (
-  UPDATE pgflow.step_states
-  SET status = 'started'
-  FROM ready_steps
-  WHERE pgflow.step_states.run_id = start_ready_steps.run_id
-    AND pgflow.step_states.step_slug = ready_steps.step_slug
-  RETURNING pgflow.step_states.*
-),
-sent_messages AS (
-  SELECT
-    started_step.flow_slug,
-    started_step.run_id,
-    started_step.step_slug,
-    pgmq.send(started_step.flow_slug, jsonb_build_object(
-      'flow_slug', started_step.flow_slug,
-      'run_id', started_step.run_id,
-      'step_slug', started_step.step_slug,
-      'task_index', 0
-    )) AS msg_id
-  FROM started_step_states AS started_step
-)
-INSERT INTO pgflow.step_tasks (flow_slug, run_id, step_slug, message_id)
-SELECT
-  sent_messages.flow_slug,
-  sent_messages.run_id,
-  sent_messages.step_slug,
-  sent_messages.msg_id
-FROM sent_messages;
-
-$$;

package/dist/supabase/migrations/000020_start_flow.sql

@@ -1,46 +0,0 @@
--- drop function if exists pgflow.start_flow;
-create or replace function pgflow.start_flow(
-  flow_slug TEXT,
-  input JSONB
-)
-returns setof PGFLOW.RUNS
-language plpgsql
-set search_path to ''
-volatile
-as $$
-declare
-  v_created_run pgflow.runs%ROWTYPE;
-begin
-
-WITH
-  flow_steps AS (
-    SELECT steps.flow_slug, steps.step_slug, steps.deps_count
-    FROM pgflow.steps
-    WHERE steps.flow_slug = start_flow.flow_slug
-  ),
-  created_run AS (
-    INSERT INTO pgflow.runs (flow_slug, input, remaining_steps)
-    VALUES (
-      start_flow.flow_slug,
-      start_flow.input,
-      (SELECT count(*) FROM flow_steps)
-    )
-    RETURNING *
-  ),
-  created_step_states AS (
-    INSERT INTO pgflow.step_states (flow_slug, run_id, step_slug, remaining_deps)
-    SELECT
-      fs.flow_slug,
-      (SELECT run_id FROM created_run),
-      fs.step_slug,
-      fs.deps_count
-    FROM flow_steps fs
-  )
-SELECT * FROM created_run INTO v_created_run;
-
-PERFORM pgflow.start_ready_steps(v_created_run.run_id);
-
-RETURN QUERY SELECT * FROM pgflow.runs where run_id = v_created_run.run_id;
-
-end;
-$$;

package/dist/supabase/migrations/000030_read_with_poll_backport.sql

@@ -1,70 +0,0 @@
---------------------------------------------------------------------------------
--- Read With Poll --------------------------------------------------------------
---                                                                            --
--- This is a backport of the pgmq.read_with_poll function from version 1.5.0  --
--- It is required because it fixes a bug with high CPU usage and Supabase     --
--- is still using version 1.4.4.                                              --
---                                                                            --
--- It is slightly modified (removed headers which are not available in 1.4.1) --
---                                                                            --
--- This will be removed once Supabase upgrades to 1.5.0 or higher.            --
---------------------------------------------------------------------------------
-create function pgflow.read_with_poll(
-  queue_name TEXT,
-  vt INTEGER,
-  qty INTEGER,
-  max_poll_seconds INTEGER default 5,
-  poll_interval_ms INTEGER default 100,
-  conditional JSONB default '{}'
-)
-returns setof PGMQ.MESSAGE_RECORD as $$
-DECLARE
-    r pgmq.message_record;
-    stop_at TIMESTAMP;
-    sql TEXT;
-    qtable TEXT := pgmq.format_table_name(queue_name, 'q');
-BEGIN
-    stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
-    LOOP
-      IF (SELECT clock_timestamp() >= stop_at) THEN
-        RETURN;
-      END IF;
-
-      sql := FORMAT(
-          $QUERY$
-          WITH cte AS
-          (
-              SELECT msg_id
-              FROM pgmq.%I
-              WHERE vt <= clock_timestamp() AND CASE
-                  WHEN %L != '{}'::jsonb THEN (message @> %2$L)::integer
-                  ELSE 1
-              END = 1
-              ORDER BY msg_id ASC
-              LIMIT $1
-              FOR UPDATE SKIP LOCKED
-          )
-          UPDATE pgmq.%I m
-          SET
-              vt = clock_timestamp() + %L,
-              read_ct = read_ct + 1
-          FROM cte
-          WHERE m.msg_id = cte.msg_id
-          RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.vt, m.message;
-          $QUERY$,
-          qtable, conditional, qtable, make_interval(secs => vt)
-      );
-
-      FOR r IN
-        EXECUTE sql USING qty
-      LOOP
-        RETURN NEXT r;
-      END LOOP;
-      IF FOUND THEN
-        RETURN;
-      ELSE
-        PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
-      END IF;
-    END LOOP;
-END;
-$$ language plpgsql;

package/dist/supabase/migrations/000040_poll_for_tasks.sql

@@ -1,100 +0,0 @@
-create or replace function pgflow.poll_for_tasks(
-  queue_name text,
-  vt integer,
-  qty integer,
-  max_poll_seconds integer default 5,
-  poll_interval_ms integer default 100
-)
-returns setof pgflow.step_task_record
-volatile
-set search_path to ''
-as $$
-
-with read_messages as (
-  select *
-  from pgflow.read_with_poll(
-    queue_name,
-    vt,
-    qty,
-    max_poll_seconds,
-    poll_interval_ms
-  )
-),
-tasks as (
-  select
-    task.flow_slug,
-    task.run_id,
-    task.step_slug,
-    task.task_index,
-    task.message_id
-  from pgflow.step_tasks as task
-  join read_messages as message on message.msg_id = task.message_id
-  where task.message_id = message.msg_id
-    and task.status = 'queued'
-),
-increment_attempts as (
-  update pgflow.step_tasks
-  set attempts_count = attempts_count + 1
-  from tasks
-  where step_tasks.message_id = tasks.message_id
-  and status = 'queued'
-),
-runs as (
-  select
-    r.run_id,
-    r.input
-  from pgflow.runs r
-  where r.run_id in (select run_id from tasks)
-),
-deps as (
-  select
-    st.run_id,
-    st.step_slug,
-    dep.dep_slug,
-    dep_task.output as dep_output
-  from tasks st
-  join pgflow.deps dep on dep.flow_slug = st.flow_slug and dep.step_slug = st.step_slug
-  join pgflow.step_tasks dep_task on
-    dep_task.run_id = st.run_id and
-    dep_task.step_slug = dep.dep_slug and
-    dep_task.status = 'completed'
-),
-deps_outputs as (
-  select
-    d.run_id,
-    d.step_slug,
-    jsonb_object_agg(d.dep_slug, d.dep_output) as deps_output
-  from deps d
-  group by d.run_id, d.step_slug
-),
-timeouts as (
-  select
-    task.message_id,
-    coalesce(step.opt_timeout, flow.opt_timeout) + 2 as vt_delay
-  from tasks task
-  join pgflow.flows flow on flow.flow_slug = task.flow_slug
-  join pgflow.steps step on step.flow_slug = task.flow_slug and step.step_slug = task.step_slug
-)
-
-select
-  st.flow_slug,
-  st.run_id,
-  st.step_slug,
-  jsonb_build_object('run', r.input) ||
-  coalesce(dep_out.deps_output, '{}'::jsonb) as input,
-  st.message_id as msg_id
-from tasks st
-join runs r on st.run_id = r.run_id
-left join deps_outputs dep_out on
-  dep_out.run_id = st.run_id and
-  dep_out.step_slug = st.step_slug
-cross join lateral (
-  -- TODO: this is slow because it calls set_vt for each row, and set_vt
-  --       builds dynamic query from string every time it is called
-  --       implement set_vt_batch(msgs_ids bigint[], vt_delays int[])
-  select pgmq.set_vt(queue_name, st.message_id,
-    (select t.vt_delay from timeouts t where t.message_id = st.message_id)
-  )
-) set_vt;
-
-$$ language sql;

package/dist/supabase/migrations/000045_maybe_complete_run.sql

@@ -1,30 +0,0 @@
-create or replace function pgflow.maybe_complete_run(run_id uuid)
-returns void
-language sql
-volatile
-set search_path to ''
-as $$
-  -- Update run status to completed and set output when there are no remaining steps
-  -- All done in a single declarative SQL statement
-  UPDATE pgflow.runs
-  SET
-    status = 'completed',
-    output = (
-      -- Get outputs from final steps (steps that are not dependencies for other steps)
-      SELECT jsonb_object_agg(st.step_slug, st.output)
-      FROM pgflow.step_tasks st
-      JOIN pgflow.step_states ss ON ss.run_id = st.run_id AND ss.step_slug = st.step_slug
-      JOIN pgflow.runs r ON r.run_id = ss.run_id AND r.flow_slug = ss.flow_slug
-      WHERE st.run_id = maybe_complete_run.run_id
-        AND st.status = 'completed'
-        AND NOT EXISTS (
-          SELECT 1
-          FROM pgflow.deps d
-          WHERE d.flow_slug = ss.flow_slug
-            AND d.dep_slug = ss.step_slug
-        )
-    )
-  WHERE pgflow.runs.run_id = maybe_complete_run.run_id
-    AND pgflow.runs.remaining_steps = 0
-    AND pgflow.runs.status != 'completed';
-$$;

package/dist/supabase/migrations/000050_complete_task.sql

@@ -1,98 +0,0 @@
--- drop function if exists pgflow.complete_task(uuid, text, int, jsonb);
-create or replace function pgflow.complete_task(
-  run_id uuid,
-  step_slug text,
-  task_index int,
-  output jsonb
-)
-returns setof pgflow.step_tasks
-language plpgsql
-volatile
-set search_path to ''
-as $$
-begin
-
-WITH run_lock AS (
-  SELECT * FROM pgflow.runs
-  WHERE pgflow.runs.run_id = complete_task.run_id
-  FOR UPDATE
-),
-step_lock AS (
-  SELECT * FROM pgflow.step_states
-  WHERE pgflow.step_states.run_id = complete_task.run_id
-    AND pgflow.step_states.step_slug = complete_task.step_slug
-  FOR UPDATE
-),
-task AS (
-  UPDATE pgflow.step_tasks
-  SET
-    status = 'completed',
-    output = complete_task.output
-  WHERE pgflow.step_tasks.run_id = complete_task.run_id
-    AND pgflow.step_tasks.step_slug = complete_task.step_slug
-    AND pgflow.step_tasks.task_index = complete_task.task_index
-  RETURNING *
-),
-step_state AS (
-  UPDATE pgflow.step_states
-  SET
-    status = CASE
-    WHEN pgflow.step_states.remaining_tasks = 1 THEN 'completed'  -- Will be 0 after decrement
-    ELSE 'started'
-    END,
-    remaining_tasks = pgflow.step_states.remaining_tasks - 1
-  FROM task
-  WHERE pgflow.step_states.run_id = complete_task.run_id
-    AND pgflow.step_states.step_slug = complete_task.step_slug
-  RETURNING pgflow.step_states.*
-),
--- Find all dependent steps if the current step was completed
-dependent_steps AS (
-  SELECT d.step_slug AS dependent_step_slug
-  FROM pgflow.deps d
-  JOIN step_state s ON s.status = 'completed' AND d.flow_slug = s.flow_slug
-  WHERE d.dep_slug = complete_task.step_slug
-  ORDER BY d.step_slug  -- Ensure consistent ordering
-),
--- Lock dependent steps before updating
-dependent_steps_lock AS (
-  SELECT * FROM pgflow.step_states
-  WHERE pgflow.step_states.run_id = complete_task.run_id
-    AND pgflow.step_states.step_slug IN (SELECT dependent_step_slug FROM dependent_steps)
-  FOR UPDATE
-),
--- Update all dependent steps
-dependent_steps_update AS (
-  UPDATE pgflow.step_states
-  SET remaining_deps = pgflow.step_states.remaining_deps - 1
-  FROM dependent_steps
-  WHERE pgflow.step_states.run_id = complete_task.run_id
-    AND pgflow.step_states.step_slug = dependent_steps.dependent_step_slug
-)
--- Only decrement remaining_steps, don't update status
-UPDATE pgflow.runs
-SET remaining_steps = pgflow.runs.remaining_steps - 1
-FROM step_state
-WHERE pgflow.runs.run_id = complete_task.run_id
-  AND step_state.status = 'completed';
-
-PERFORM pgmq.archive(
-  queue_name => (SELECT run.flow_slug FROM pgflow.runs AS run WHERE run.run_id = complete_task.run_id),
-  msg_id => (SELECT message_id FROM pgflow.step_tasks AS step_task
-             WHERE step_task.run_id = complete_task.run_id
-             AND step_task.step_slug = complete_task.step_slug
-             AND step_task.task_index = complete_task.task_index)
-);
-
-PERFORM pgflow.start_ready_steps(complete_task.run_id);
-
-PERFORM pgflow.maybe_complete_run(complete_task.run_id);
-
-RETURN QUERY SELECT *
-FROM pgflow.step_tasks AS step_task
-WHERE step_task.run_id = complete_task.run_id
-  AND step_task.step_slug = complete_task.step_slug
-  AND step_task.task_index = complete_task.task_index;
-
-end;
-$$;

package/dist/supabase/migrations/000055_calculate_retry_delay.sql

@@ -1,11 +0,0 @@
-create or replace function pgflow.calculate_retry_delay(
-  base_delay numeric,
-  attempts_count int
-)
-returns int
-language sql
-immutable
-parallel safe
-as $$
-  select floor(base_delay * power(2, attempts_count))::int
-$$;