@pgflow/edge-worker 0.0.5 → 0.0.7-prealpha.1

package/supabase/functions/creating_queue/index.ts

@@ -1,5 +0,0 @@
-import { EdgeWorker } from '../_src/EdgeWorker.ts';
-
-EdgeWorker.start(console.log, {
-  queueName: 'creating_queue',
-});

package/supabase/functions/failing_always/index.ts

@@ -1,13 +0,0 @@
-import { EdgeWorker } from '../_src/EdgeWorker.ts';
-
-function failingAlways() {
-  console.log('(╯°□°)╯︵ ┻━┻');
-  throw new Error('(╯°□°)╯︵ ┻━┻');
-}
-
-EdgeWorker.start(failingAlways, {
-  queueName: 'failing_always',
-  retryLimit: 2,
-  retryDelay: 2,
-  maxPollSeconds: 1,
-});

package/supabase/functions/increment_sequence/index.ts

@@ -1,14 +0,0 @@
-import { EdgeWorker } from '../_src/EdgeWorker.ts';
-import { sql } from '../utils.ts';
-
-// await sql`CREATE SEQUENCE IF NOT EXISTS test_seq`;
-// await sql`SELECT pgmq.create('increment_sequence')`;
-
-async function incrementCounter() {
-  console.log(
-    '[increment_sequence] next_seq =',
-    await sql`SELECT nextval('test_seq')`
-  );
-}
-
-EdgeWorker.start(incrementCounter, { queueName: 'increment_sequence' });

package/supabase/functions/max_concurrency/index.ts

@@ -1,17 +0,0 @@
-import { EdgeWorker } from '../_src/EdgeWorker.ts';
-import { sleep, sql } from '../utils.ts';
-
-async function incrementSeq() {
-  await sleep(50);
-
-  console.log(
-    '[max_concurrency] last_val =',
-    await sql`SELECT nextval('test_seq')`
-  );
-}
-
-EdgeWorker.start(incrementSeq, {
-  queueName: 'max_concurrency',
-  maxConcurrent: 10,
-  maxPgConnections: 4,
-});

package/supabase/functions/serial_sleep/index.ts

@@ -1,16 +0,0 @@
-import { EdgeWorker } from '../_src/EdgeWorker.ts';
-import { sql, sleep } from '../utils.ts';
-
-const sleep1s = async () => {
-  console.time('Task time');
-  const lastVal = await sql`SELECT nextval('test_seq')`;
-  console.log('[serial_sleep] lastVal =', lastVal);
-  await sleep(1000);
-  console.timeEnd('Task time');
-};
-
-EdgeWorker.start(sleep1s, {
-  queueName: 'serial_sleep',
-  maxConcurrent: 1,
-  visibilityTimeout: 5, // higher than the delay()
-});

package/supabase/functions/utils.ts

@@ -1,13 +0,0 @@
-import postgres from 'postgres';
-import { delay } from '@std/async';
-
-const EDGE_WORKER_DB_URL = Deno.env.get('EDGE_WORKER_DB_URL')!;
-console.log('EDGE_WORKER_DB_URL', EDGE_WORKER_DB_URL);
-
-export const sql = postgres(EDGE_WORKER_DB_URL, { prepare: false });
-
-export const sleep = delay;
-
-export function randomInt(min: number, max: number) {
-  return Math.floor(Math.random() * (max - min + 1)) + min;
-}

package/supabase/seed.sql

@@ -1,2 +0,0 @@
-select vault.create_secret('http://host.docker.internal:50321', 'app_url');
-select vault.create_secret('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0', 'supabase_anon_key');

package/tests/db/compose.yaml

@@ -1,20 +0,0 @@
-services:
-  db:
-    # image: postgres:15.8-alpine
-    # image: supabase/postgres:15.8.1.039
-    # image: supabase/postgres:15.1.0.37
-
-    image: supabase/postgres:15.8.1.020
-    # image: supabase/postgres:15.8.1.018
-    ports:
-      - '5432:5432'
-    volumes:
-      - ./migrations/edge_worker.sql:/docker-entrypoint-initdb.d/migrations/950_edge_worker.sql
-    environment:
-      POSTGRES_DB: postgres
-      # commented out because of the problems with supabase_admin permissions
-      # see: https://github.com/supabase/postgres/issues/1219#issuecomment-2362955730
-      # POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_HOST: /var/run/postgresql
-      POSTGRES_PORT: 5432

package/tests/db.ts

@@ -1,71 +0,0 @@
-import postgres from 'postgres';
-
-function createSql(dbUrl: string) {
-  return postgres(dbUrl, {
-    prepare: false,
-    onnotice(_: unknown) {
-      // no-op to silence notices
-    },
-  });
-}
-
-export function withTransaction(
-  callback: (sql: postgres.Sql) => Promise<unknown>
-) {
-  const dbUrl = `postgresql://supabase_admin:postgres@localhost:5432/postgres`;
-  const localSql = createSql(dbUrl);
-
-  return async () => {
-    try {
-      console.log('calling callback');
-
-      let callbackError: unknown = null;
-
-      await localSql.begin(async (sql: postgres.Sql) => {
-        // Add no-op end() method to transaction-local sql
-        const wrappedSql = Object.assign(sql, {
-          end: async () => {
-            /* no-op */
-          },
-        });
-
-        try {
-          await callback(wrappedSql);
-        } catch (error) {
-          callbackError = error;
-        } finally {
-          console.log('Rolling back transaction');
-          // Using ROLLBACK AND CHAIN to avoid "no transaction in progress" warning
-          await sql`ROLLBACK AND CHAIN`;
-        }
-      });
-
-      console.log('callback called');
-
-      if (callbackError) {
-        throw callbackError;
-      }
-    } catch (err) {
-      console.error('Error in withTransaction:', err);
-      throw err;
-    } finally {
-      console.log('Closing connection');
-      await localSql.end();
-    }
-  };
-}
-
-export function withPgNoTransaction(
-  callback: (sql: postgres.Sql) => Promise<unknown>
-) {
-  const dbUrl = 'postgresql://supabase_admin:postgres@localhost:5432/postgres';
-  const sql = createSql(dbUrl);
-
-  return async () => {
-    try {
-      await callback(sql);
-    } finally {
-      await sql.end();
-    }
-  };
-}

package/tests/e2e/README.md

@@ -1,54 +0,0 @@
-# E2E Testing Strategy
-
-We use a real Supabase instance running locally (in `supabase/`) to perform end-to-end testing of the entire EdgeWorker stack. The tests interact with purpose-built test workers that simulate different behaviors:
-
-- Different queues with specific behaviors
-- Workers that always fail
-- Workers with varying retry patterns
-- etc.
-
-Lot of workers increment `test_seq` as a mean to identify the completion of a job.
-This also allows to identify how many times handlers were called if any retries were attempted.
-
-The test flow is straightforward:
-
-1. Put messages on specific queues
-2. Worker calls handlers, handlers increment `test_seq`
-3. `await` until `test_seq` was incremented to expected value (number of messages)
-4. Assert that workers behaved as expected (retries, failures, etc.)
-
-This approach lets us verify the entire stack from message enqueueing through worker processing, retries, and completion.
-
-## Core Test Scenarios Needed
-
-#### Glossary
-
-- `worker` - instance of given worker edge function that is subject to CPU and memory limits and can be killed
-- `worker function` - edge function within supabase app that uses EdgeWorker instead of serving requests
-- `queue` - pgmq queue that workers can pull from
-- `message` - PGMQ `message_record` that contains metadata (`msg_id`, `read_ct`, `vt`) and payload (`message JSONB`)
-
-### [ ] Happy Path
-
-- [x] Worker picks messages from queue
-- [ ] Worker calls handler function with each message
-- [x] Worker can process big amounts of messages (restarts itself when CPU clock limit hits)
-- [x] Different worker functions can pull from different queues
-- [ ] Different worker functions can pull from the same queue
-
-### [x] Worker Lifecycle
-
-- [x] Worker registers on start
-- [x] Worker sends heartbeats every 5s
-- [x] Worker updates function_name with heartbeat
-
-### [ ] Retries & Failures
-
-- [ ] Worker retries failed jobs n-times and succeeds
-- [ ] Worker uses exponential backoff for each subsequent retry
-- [x] Worker uses proper number of retries for each job
-- [x] Worker archives jobs that will not be retried
-
-### [x] Concurrency
-
-- [x] Worker respects maxConcurrent and processes messages in serial when set to 1

package/tests/e2e/_helpers.ts

@@ -1,135 +0,0 @@
-import { sql } from '../sql.ts';
-import { delay } from '@std/async';
-import ProgressBar from 'jsr:@deno-library/progress';
-import { dim } from 'https://deno.land/std@0.224.0/fmt/colors.ts';
-
-interface WaitForOptions {
-  pollIntervalMs?: number;
-  timeoutMs?: number;
-  description?: string;
-}
-
-export function log(message: string, ...args: unknown[]) {
-  console.log(dim(` -> ${message}`), ...args);
-}
-
-export async function waitFor<T>(
-  predicate: () => Promise<T | false>,
-  options: WaitForOptions = {}
-): Promise<T> {
-  const {
-    pollIntervalMs = 250,
-    timeoutMs = 30000,
-    description = 'condition',
-  } = options;
-
-  const startTime = Date.now();
-
-  while (true) {
-    const result = await predicate();
-
-    if (result) return result;
-
-    if (Date.now() - startTime > timeoutMs) {
-      throw new Error(
-        `Timeout after ${timeoutMs}ms waiting for ${description}`
-      );
-    }
-
-    await delay(pollIntervalMs);
-  }
-}
-
-export async function sendBatch(count: number, queueName: string) {
-  return await sql`
-    SELECT pgmq.send_batch(
-      ${queueName},
-      ARRAY(
-        SELECT '{}'::jsonb
-        FROM generate_series(1, ${count}::integer)
-      )
-    )`;
-}
-
-export async function seqLastValue(
-  seqName: string = 'test_seq'
-): Promise<number> {
-  // Postgres sequences are initialized with a value of 1,
-  // but incrementing them for the first time does not increment the last_value,
-  // only sets is_called to true
-  const seqResult = await sql`
-    SELECT
-      CASE
-        WHEN is_called THEN last_value::integer
-        ELSE 0
-      END as last_value
-    FROM ${sql(seqName)}`;
-  return seqResult[0].last_value;
-}
-
-interface WaitForSeqValueOptions {
-  pollIntervalMs?: number;
-  seqName?: string;
-  timeoutMs?: number;
-}
-
-export async function waitForSeqToIncrementBy(
-  value: number,
-  options: WaitForSeqValueOptions = {}
-): Promise<number> {
-  const { seqName = 'test_seq' } = options;
-
-  const perSecond = 0;
-
-  const progress = new ProgressBar({
-    title: `${seqName} (${perSecond}/s)`,
-    total: value,
-    width: 20,
-    display: dim(
-      ` -> incrementing "${seqName}": :completed/:total (:eta left) [:bar] :percent`
-    ),
-    prettyTime: true,
-  });
-
-  const startVal = await seqLastValue(seqName);
-  let lastVal = startVal;
-
-  return await waitFor(
-    async () => {
-      lastVal = await seqLastValue(seqName);
-      progress.render(lastVal);
-      const incrementedBy = lastVal - startVal;
-
-      return incrementedBy >= value ? lastVal : false;
-    },
-    {
-      ...options,
-      description: `sequence ${seqName} to reach value ${value}`,
-    }
-  );
-}
-
-export async function waitForActiveWorker() {
-  return await waitFor(
-    async () => {
-      const [{ has_active: hasActiveWorker }] =
-        await sql`SELECT count(*) > 0 AS has_active FROM edge_worker.active_workers`;
-      log('waiting for active worker ', hasActiveWorker);
-      return hasActiveWorker;
-    },
-    {
-      pollIntervalMs: 300,
-      description: 'active worker',
-    }
-  );
-}
-
-export async function fetchWorkers(functionName: string) {
-  return await sql`SELECT * FROM edge_worker.workers WHERE function_name = ${functionName}`;
-}
-
-export async function startWorker(workerName: string) {
-  await sql`SELECT edge_worker.spawn(${workerName}::text)`;
-  await waitForActiveWorker();
-  log('worker spawned!');
-}

package/tests/e2e/performance.test.ts

@@ -1,60 +0,0 @@
-import { sql } from '../sql.ts';
-import {
-  waitFor,
-  sendBatch,
-  waitForSeqToIncrementBy,
-  startWorker,
-  log,
-} from './_helpers.ts';
-
-const MESSAGES_TO_SEND = 20000;
-const WORKER_NAME = 'max_concurrency';
-
-Deno.test(
-  'worker can handle tens of thousands of jobs queued at once',
-  async () => {
-    await sql`CREATE SEQUENCE IF NOT EXISTS test_seq`;
-    await sql`ALTER SEQUENCE test_seq RESTART WITH 1`;
-    await sql`SELECT pgmq.create(${WORKER_NAME})`;
-    await sql`SELECT pgmq.drop_queue(${WORKER_NAME})`;
-    await sql`SELECT pgmq.create(${WORKER_NAME})`;
-    await startWorker(WORKER_NAME);
-    await waitFor(
-      async () => {
-        const [{ worker_count }] = await sql`
-        SELECT COUNT(*)::integer AS worker_count
-        FROM edge_worker.active_workers
-        WHERE function_name = ${WORKER_NAME}
-      `;
-
-        log('worker_count', worker_count);
-        return worker_count === 1;
-      },
-      { description: 'Waiting for exacly one worker' }
-    );
-
-    try {
-      // worker sleeps for 1s for each message
-      // se we will expect roughly 1 message per second
-      const startTime = Date.now();
-
-      await sendBatch(MESSAGES_TO_SEND, WORKER_NAME);
-      await waitForSeqToIncrementBy(MESSAGES_TO_SEND, {
-        timeoutMs: MESSAGES_TO_SEND * 1000 + 1000,
-        pollIntervalMs: 1000,
-      });
-
-      const endTime = Date.now();
-      const totalMs = Math.round(endTime - startTime);
-      const totalS = totalMs / 1000;
-      const msgsPerSecond = MESSAGES_TO_SEND / totalS;
-
-      log('');
-      log('');
-      log(`Total time:`, totalMs);
-      log(`msgs/second:`, msgsPerSecond);
-    } finally {
-      await sql.end();
-    }
-  }
-);

package/tests/e2e/restarts.test.ts

@@ -1,56 +0,0 @@
-import { sql } from '../sql.ts';
-import { assertGreater, assertGreaterOrEqual } from 'jsr:@std/assert';
-import {
-  fetchWorkers,
-  sendBatch,
-  seqLastValue,
-  startWorker,
-  waitForSeqToIncrementBy,
-} from './_helpers.ts';
-
-const WORKER_NAME = 'cpu_intensive';
-
-// TODO: document relation between CPU clock limit, amount of time to process
-//       single message and amount of messages to send
-const MESSAGES_TO_SEND = 30;
-
-Deno.test('should spawn next worker when CPU clock limit hits', async () => {
-  await sql`CREATE SEQUENCE IF NOT EXISTS test_seq`;
-  await sql`ALTER SEQUENCE test_seq RESTART WITH 1`;
-  try {
-    await sql`SELECT pgmq.drop_queue(${WORKER_NAME})`;
-  } catch {
-    // ignore
-  }
-  await sql`SELECT pgmq.create(${WORKER_NAME})`;
-  await sql`
-    DELETE FROM edge_worker.workers
-    WHERE worker_id IN (
-      SELECT worker_id
-      FROM edge_worker.inactive_workers
-    )`;
-  await startWorker(WORKER_NAME);
-
-  try {
-    await sendBatch(MESSAGES_TO_SEND, WORKER_NAME);
-    await waitForSeqToIncrementBy(MESSAGES_TO_SEND, {
-      timeoutMs: 35000,
-      pollIntervalMs: 300,
-    });
-
-    assertGreaterOrEqual(
-      await seqLastValue(),
-      MESSAGES_TO_SEND,
-      'Sequence value should be greater than or equal to the number of messages sent'
-    );
-
-    const workers = await fetchWorkers(WORKER_NAME);
-    assertGreater(
-      workers.length,
-      1,
-      'expected worker to spawn another but there is only 1 worker'
-    );
-  } finally {
-    await sql.end();
-  }
-});

package/tests/helpers.ts

@@ -1,22 +0,0 @@
-import type { PgmqMessageRecord } from "../src/queue/types.ts";
-import type { postgres } from "./sql.ts";
-
-export async function sendBatch(count: number, queueName: string, sql: postgres.Sql) {
-  return await sql`
-    SELECT pgmq.send_batch(
-      ${queueName},
-      ARRAY(
-        SELECT '{}'::jsonb
-        FROM generate_series(1, ${count}::integer)
-      )
-    )`;
-}
-
-/**
- * Fetches archived messages from the queue
- */
-export async function getArchivedMessages(sql: postgres.Sql, queueName: string) {
-  return await sql<PgmqMessageRecord[]>
-    `SELECT * FROM ${sql('pgmq.a_' + queueName)}`;
-}
-

package/tests/integration/_helpers.ts

@@ -1,43 +0,0 @@
-import type { AnyFlow, ExtractFlowInput, Flow, Json } from '@pgflow/dsl';
-import {
-  createFlowWorker,
-  type FlowWorkerConfig,
-} from '../../src/flow/createFlowWorker.ts';
-import type { postgres } from '../sql.ts';
-import { PgflowSqlClient } from '../../../core/src/PgflowSqlClient.ts';
-
-export async function startFlow<TFlow extends AnyFlow>(
-  sql: postgres.Sql,
-  flow: TFlow,
-  input: ExtractFlowInput<TFlow>
-) {
-  const pgflow = new PgflowSqlClient<TFlow>(sql);
-
-  return await pgflow.startFlow(flow, input);
-}
-
-export function startWorker<
-  T extends Json,
-  S extends Record<string, Json> = Record<never, never>,
-  D extends Record<string, string[]> = Record<string, string[]>
->(sql: postgres.Sql, flow: Flow<T, S, D>, options: FlowWorkerConfig) {
-  const defaultOptions = {
-    sql,
-    maxConcurrent: 1,
-    batchSize: 10,
-  };
-
-  const mergedOptions = {
-    ...defaultOptions,
-    ...options,
-  };
-
-  const worker = createFlowWorker(flow, mergedOptions);
-
-  worker.startOnlyOnce({
-    edgeFunctionName: 'test_flow',
-    workerId: crypto.randomUUID(),
-  });
-
-  return worker;
-}

package/tests/integration/creating_queue.test.ts

@@ -1,32 +0,0 @@
-import { assertEquals } from "@std/assert";
-import { createQueueWorker } from '../../src/queue/createQueueWorker.ts';
-import { withTransaction } from "../db.ts";
-import { delay } from "@std/async";
-
-Deno.test('creates queue when starting worker', withTransaction(async (sql) => {
-  const worker = createQueueWorker(console.log, {
-    sql,
-    maxPollSeconds: 1,
-    queueName: 'custom_queue'
-  });
-
-  worker.startOnlyOnce({
-    edgeFunctionName: 'test',
-    // random uuid
-    workerId: crypto.randomUUID(),
-  });
-
-  await delay(100);
-
-  try {
-    const result: {queue_name: string}[] = await sql`select queue_name from pgmq.list_queues();`;
-
-    assertEquals(
-      [...result],
-      [{ queue_name: 'custom_queue' }],
-      'queue "custom_queue" was created'
-    );
-  } finally {
-    await worker.stop();
-  }
-}));