@pgflow/core 0.0.21 → 0.0.22

package/dist/supabase/migrations/000050_complete_task.sql

@@ -0,0 +1,98 @@
+-- drop function if exists pgflow.complete_task(uuid, text, int, jsonb);
+create or replace function pgflow.complete_task(
+  run_id uuid,
+  step_slug text,
+  task_index int,
+  output jsonb
+)
+returns setof pgflow.step_tasks
+language plpgsql
+volatile
+set search_path to ''
+as $$
+begin
+
+WITH run_lock AS (
+  SELECT * FROM pgflow.runs
+  WHERE pgflow.runs.run_id = complete_task.run_id
+  FOR UPDATE
+),
+step_lock AS (
+  SELECT * FROM pgflow.step_states
+  WHERE pgflow.step_states.run_id = complete_task.run_id
+    AND pgflow.step_states.step_slug = complete_task.step_slug
+  FOR UPDATE
+),
+task AS (
+  UPDATE pgflow.step_tasks
+  SET
+    status = 'completed',
+    output = complete_task.output
+  WHERE pgflow.step_tasks.run_id = complete_task.run_id
+    AND pgflow.step_tasks.step_slug = complete_task.step_slug
+    AND pgflow.step_tasks.task_index = complete_task.task_index
+  RETURNING *
+),
+step_state AS (
+  UPDATE pgflow.step_states
+  SET
+    status = CASE
+    WHEN pgflow.step_states.remaining_tasks = 1 THEN 'completed'  -- Will be 0 after decrement
+    ELSE 'started'
+    END,
+    remaining_tasks = pgflow.step_states.remaining_tasks - 1
+  FROM task
+  WHERE pgflow.step_states.run_id = complete_task.run_id
+    AND pgflow.step_states.step_slug = complete_task.step_slug
+  RETURNING pgflow.step_states.*
+),
+-- Find all dependent steps if the current step was completed
+dependent_steps AS (
+  SELECT d.step_slug AS dependent_step_slug
+  FROM pgflow.deps d
+  JOIN step_state s ON s.status = 'completed' AND d.flow_slug = s.flow_slug
+  WHERE d.dep_slug = complete_task.step_slug
+  ORDER BY d.step_slug  -- Ensure consistent ordering
+),
+-- Lock dependent steps before updating
+dependent_steps_lock AS (
+  SELECT * FROM pgflow.step_states
+  WHERE pgflow.step_states.run_id = complete_task.run_id
+    AND pgflow.step_states.step_slug IN (SELECT dependent_step_slug FROM dependent_steps)
+  FOR UPDATE
+),
+-- Update all dependent steps
+dependent_steps_update AS (
+  UPDATE pgflow.step_states
+  SET remaining_deps = pgflow.step_states.remaining_deps - 1
+  FROM dependent_steps
+  WHERE pgflow.step_states.run_id = complete_task.run_id
+    AND pgflow.step_states.step_slug = dependent_steps.dependent_step_slug
+)
+-- Only decrement remaining_steps, don't update status
+UPDATE pgflow.runs
+SET remaining_steps = pgflow.runs.remaining_steps - 1
+FROM step_state
+WHERE pgflow.runs.run_id = complete_task.run_id
+  AND step_state.status = 'completed';
+
+PERFORM pgmq.archive(
+  queue_name => (SELECT run.flow_slug FROM pgflow.runs AS run WHERE run.run_id = complete_task.run_id),
+  msg_id => (SELECT message_id FROM pgflow.step_tasks AS step_task
+             WHERE step_task.run_id = complete_task.run_id
+             AND step_task.step_slug = complete_task.step_slug
+             AND step_task.task_index = complete_task.task_index)
+);
+
+PERFORM pgflow.start_ready_steps(complete_task.run_id);
+
+PERFORM pgflow.maybe_complete_run(complete_task.run_id);
+
+RETURN QUERY SELECT *
+FROM pgflow.step_tasks AS step_task
+WHERE step_task.run_id = complete_task.run_id
+  AND step_task.step_slug = complete_task.step_slug
+  AND step_task.task_index = complete_task.task_index;
+
+end;
+$$;

package/dist/supabase/migrations/000055_calculate_retry_delay.sql

@@ -0,0 +1,11 @@
+create or replace function pgflow.calculate_retry_delay(
+  base_delay numeric,
+  attempts_count int
+)
+returns int
+language sql
+immutable
+parallel safe
+as $$
+  select floor(base_delay * power(2, attempts_count))::int
+$$;

package/dist/supabase/migrations/000060_fail_task.sql

@@ -0,0 +1,124 @@
+create or replace function pgflow.fail_task(
+  run_id uuid,
+  step_slug text,
+  task_index int,
+  error_message text
+)
+returns setof pgflow.step_tasks
+language plpgsql
+volatile
+set search_path to ''
+as $$
+begin
+
+WITH run_lock AS (
+  SELECT * FROM pgflow.runs
+  WHERE pgflow.runs.run_id = fail_task.run_id
+  FOR UPDATE
+),
+step_lock AS (
+  SELECT * FROM pgflow.step_states
+  WHERE pgflow.step_states.run_id = fail_task.run_id
+    AND pgflow.step_states.step_slug = fail_task.step_slug
+  FOR UPDATE
+),
+flow_info AS (
+  SELECT r.flow_slug
+  FROM pgflow.runs r
+  WHERE r.run_id = fail_task.run_id
+),
+config AS (
+  SELECT
+    COALESCE(s.opt_max_attempts, f.opt_max_attempts) AS opt_max_attempts,
+    COALESCE(s.opt_base_delay, f.opt_base_delay) AS opt_base_delay
+  FROM pgflow.steps s
+  JOIN pgflow.flows f ON f.flow_slug = s.flow_slug
+  JOIN flow_info fi ON fi.flow_slug = s.flow_slug
+  WHERE s.flow_slug = fi.flow_slug AND s.step_slug = fail_task.step_slug
+),
+
+fail_or_retry_task as (
+  UPDATE pgflow.step_tasks as task
+  SET
+    status = CASE
+      WHEN task.attempts_count < (SELECT opt_max_attempts FROM config) THEN 'queued'
+      ELSE 'failed'
+    END,
+    error_message = fail_task.error_message
+  WHERE task.run_id = fail_task.run_id
+    AND task.step_slug = fail_task.step_slug
+    AND task.task_index = fail_task.task_index
+    AND task.status = 'queued'
+  RETURNING *
+),
+maybe_fail_step AS (
+  UPDATE pgflow.step_states
+  SET
+    status = CASE
+             WHEN (select fail_or_retry_task.status from fail_or_retry_task) = 'failed' THEN 'failed'
+             ELSE pgflow.step_states.status
+             END
+  FROM fail_or_retry_task
+  WHERE pgflow.step_states.run_id = fail_task.run_id
+    AND pgflow.step_states.step_slug = fail_task.step_slug
+  RETURNING pgflow.step_states.*
+)
+UPDATE pgflow.runs
+SET status = CASE
+              WHEN (select status from maybe_fail_step) = 'failed' THEN 'failed'
+              ELSE status
+              END
+WHERE pgflow.runs.run_id = fail_task.run_id;
+
+-- For queued tasks: delay the message for retry with exponential backoff
+PERFORM (
+  WITH retry_config AS (
+    SELECT
+      COALESCE(s.opt_base_delay, f.opt_base_delay) AS base_delay
+    FROM pgflow.steps s
+    JOIN pgflow.flows f ON f.flow_slug = s.flow_slug
+    JOIN pgflow.runs r ON r.flow_slug = f.flow_slug
+    WHERE r.run_id = fail_task.run_id
+      AND s.step_slug = fail_task.step_slug
+  ),
+  queued_tasks AS (
+    SELECT
+      r.flow_slug,
+      st.message_id,
+      pgflow.calculate_retry_delay((SELECT base_delay FROM retry_config), st.attempts_count) AS calculated_delay
+    FROM pgflow.step_tasks st
+    JOIN pgflow.runs r ON st.run_id = r.run_id
+    WHERE st.run_id = fail_task.run_id
+      AND st.step_slug = fail_task.step_slug
+      AND st.task_index = fail_task.task_index
+      AND st.status = 'queued'
+  )
+  SELECT pgmq.set_vt(qt.flow_slug, qt.message_id, qt.calculated_delay)
+  FROM queued_tasks qt
+  WHERE EXISTS (SELECT 1 FROM queued_tasks)
+);
+
+-- For failed tasks: archive the message
+PERFORM (
+  WITH failed_tasks AS (
+    SELECT r.flow_slug, st.message_id
+    FROM pgflow.step_tasks st
+    JOIN pgflow.runs r ON st.run_id = r.run_id
+    WHERE st.run_id = fail_task.run_id
+      AND st.step_slug = fail_task.step_slug
+      AND st.task_index = fail_task.task_index
+      AND st.status = 'failed'
+  )
+  SELECT pgmq.archive(ft.flow_slug, ft.message_id)
+  FROM failed_tasks ft
+  WHERE EXISTS (SELECT 1 FROM failed_tasks)
+);
+
+return query select *
+from pgflow.step_tasks st
+where st.run_id = fail_task.run_id
+  and st.step_slug = fail_task.step_slug
+  and st.task_index = fail_task.task_index;
+
+end;
+$$;

package/dist/supabase/migrations/000_edge_worker_initial.sql

@@ -0,0 +1,86 @@
+create extension if not exists pgmq version '1.4.4';
+
+create schema if not exists edge_worker;
+
+-------------------------------------------------------------------------------
+-- Workers Table --------------------------------------------------------------
+-------------------------------------------------------------------------------
+create table if not exists edge_worker.workers (
+  worker_id uuid not null primary key,
+  queue_name text not null,
+  function_name text not null,
+  started_at timestamptz not null default now(),
+  stopped_at timestamptz,
+  last_heartbeat_at timestamptz not null default now()
+);
+
+--------------------------------------------------------------------------------
+-- Read With Poll --------------------------------------------------------------
+--                                                                            --
+-- This is a backport of the pgmq.read_with_poll function from version 1.5.0  --
+-- It is required because it fixes a bug with high CPU usage and Supabase     --
+-- is still using version 1.4.4.                                              --
+--                                                                            --
+-- It is slightly modified (removed headers which are not available in 1.4.1) --
+--                                                                            --
+-- This will be removed once Supabase upgrades to 1.5.0 or higher.            --
+--------------------------------------------------------------------------------
+create function edge_worker.read_with_poll(
+  queue_name text,
+  vt integer,
+  qty integer,
+  max_poll_seconds integer default 5,
+  poll_interval_ms integer default 100,
+  conditional jsonb default '{}'
+)
+returns setof pgmq.message_record as $$
+DECLARE
+    r pgmq.message_record;
+    stop_at timestamp;
+    sql text;
+    qtable text := pgmq.format_table_name(queue_name, 'q');
+BEGIN
+    stop_at := clock_timestamp() + make_interval(secs => max_poll_seconds);
+    LOOP
+      IF (SELECT clock_timestamp() >= stop_at) THEN
+        RETURN;
+      END IF;
+
+      sql := FORMAT(
+          $QUERY$
+          WITH cte AS
+          (
+              SELECT msg_id
+              FROM pgmq.%I
+              WHERE vt <= clock_timestamp() AND CASE
+                  WHEN %L != '{}'::jsonb THEN (message @> %2$L)::integer
+                  ELSE 1
+              END = 1
+              ORDER BY msg_id ASC
+              LIMIT $1
+              FOR UPDATE SKIP LOCKED
+          )
+          UPDATE pgmq.%I m
+          SET
+              vt = clock_timestamp() + %L,
+              read_ct = read_ct + 1
+          FROM cte
+          WHERE m.msg_id = cte.msg_id
+          RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.vt, m.message;
+          $QUERY$,
+          qtable, conditional, qtable, make_interval(secs => vt)
+      );
+
+      FOR r IN
+        EXECUTE sql USING qty
+      LOOP
+        RETURN NEXT r;
+      END LOOP;
+      IF FOUND THEN
+        RETURN;
+      ELSE
+        PERFORM pg_sleep(poll_interval_ms::numeric / 1000);
+      END IF;
+    END LOOP;
+END;
+$$ language plpgsql;

package/dist/tsconfig.lib.tsbuildinfo

@@ -0,0 +1 @@
+{"version":"5.6.3"}

package/dist/types.d.ts

@@ -0,0 +1,77 @@
+import type { ExtractFlowSteps, StepInput, Simplify, AnyFlow } from '@pgflow/dsl';
+import type { Database } from './database-types.js';
+export type Json = string | number | boolean | null | {
+    [key: string]: Json | undefined;
+} | Json[];
+/**
+ * Record representing a task from pgflow.poll_for_tasks
+ *
+ * Same as pgflow.step_task_record type, but with not-null fields and type argument for payload.
+ * The input type is automatically inferred based on the step_slug using a discriminated union.
+ * This ensures that each step only receives inputs from its declared dependencies and the flow's run input.
+ */
+export type StepTaskRecord<TFlow extends AnyFlow> = {
+    [StepSlug in Extract<keyof ExtractFlowSteps<TFlow>, string>]: {
+        flow_slug: string;
+        run_id: string;
+        step_slug: StepSlug;
+        input: Simplify<StepInput<TFlow, StepSlug>>;
+        msg_id: number;
+    };
+}[Extract<keyof ExtractFlowSteps<TFlow>, string>];
+/**
+ * Composite key that is enough to find a particular step task
+ * Contains only the minimum fields needed to identify a task
+ */
+export type StepTaskKey = Pick<StepTaskRecord<any>, 'run_id' | 'step_slug'>;
+/**
+ * Interface for interacting with pgflow database functions
+ */
+export interface IPgflowClient<TFlow extends AnyFlow = AnyFlow> {
+    /**
+     * Fetches tasks from pgflow
+     * @param queueName - Name
+     * @param batchSize - Number of tasks to fetch
+     * @param visibilityTimeout - Visibility timeout for tasks
+     * @param maxPollSeconds - Maximum time to poll for tasks
+     * @param pollIntervalMs - Poll interval in milliseconds
+     */
+    pollForTasks(queueName: string, batchSize?: number, visibilityTimeout?: number, maxPollSeconds?: number, pollIntervalMs?: number): Promise<StepTaskRecord<TFlow>[]>;
+    /**
+     * Marks a task as completed
+     * @param stepTask - Step task key containing run_id and step_slug
+     * @param output - Output payload for the task
+     */
+    completeTask(stepTask: StepTaskKey, output?: Json): Promise<void>;
+    /**
+     * Marks a task as failed
+     * @param stepTask - Step task key containing run_id and step_slug
+     * @param error - Error to fail task with
+     */
+    failTask(stepTask: StepTaskKey, error: unknown): Promise<void>;
+}
+/**
+ * Record representing a flow from pgflow.flows
+ */
+export type FlowRow = Database['pgflow']['Tables']['flows']['Row'];
+/**
+ * Record representing a step from pgflow.steps
+ */
+export type StepRow = Database['pgflow']['Tables']['steps']['Row'];
+/**
+ * Record representing a step from pgflow.deps
+ */
+export type DepRow = Database['pgflow']['Tables']['deps']['Row'];
+/**
+ * Record representing a step from pgflow.queues
+ */
+export type RunRow = Database['pgflow']['Tables']['runs']['Row'];
+/**
+ * Record representing a step from pgflow.step_states
+ */
+export type StepStateRow = Database['pgflow']['Tables']['step_states']['Row'];
+/**
+ * Record representing a step from pgflow.step_tasks
+ */
+export type StepTaskRow = Database['pgflow']['Tables']['step_tasks']['Row'];
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,OAAO,EACR,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,MAAM,IAAI,GACZ,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAAE,GACnC,IAAI,EAAE,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,CAAC,KAAK,SAAS,OAAO,IAAI;KACjD,QAAQ,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,GAAG;QAC5D,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,QAAQ,CAAC;QACpB,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5C,MAAM,EAAE,MAAM,CAAC;KAChB;CACF,CAAC,OAAO,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAElD;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,QAAQ,GAAG,WAAW,CAAC,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,KAAK,SAAS,OAAO,GAAG,OAAO;IAC5D;;;;;;;OAOG;IACH,YAAY,CACV,SAAS,EAAE,MAAM,EACjB,SAAS,CAAC,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,MAAM,EAC1B,cAAc,CAAC,EAAE,MAAM,EACvB,cAAc,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAEpC;;;;OAIG;IACH,YAAY,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElE;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;AAEjE;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;AAEjE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC;AAE9E;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC"}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pgflow/core",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "license": "AGPL-3.0",
   "type": "module",
   "main": "./dist/index.js",
@@ -24,7 +24,7 @@
   },
   "dependencies": {
     "postgres": "^3.4.5",
-    "@pgflow/dsl": "0.0.21"
+    "@pgflow/dsl": "0.0.22"
   },
   "publishConfig": {
     "access": "public",