@pgflow/edge-worker 0.0.5 → 0.0.7-prealpha.1

package/src/flow/StepTaskPoller.ts

@@ -1,51 +0,0 @@
-import type { StepTaskRecord, IPgflowClient } from './types.ts';
-import type { IPoller } from '../core/types.ts';
-import { getLogger } from '../core/Logger.ts';
-import type { AnyFlow } from '@pgflow/dsl';
-
-export interface StepTaskPollerConfig {
-  batchSize: number;
-  queueName: string;
-}
-
-/**
- * A poller that retrieves flow tasks using an IPgflowClient
- */
-export class StepTaskPoller<TFlow extends AnyFlow>
-  implements IPoller<StepTaskRecord<TFlow>>
-{
-  private logger = getLogger('StepTaskPoller');
-
-  constructor(
-    private readonly adapter: IPgflowClient<TFlow>,
-    private readonly signal: AbortSignal,
-    private readonly config: StepTaskPollerConfig
-  ) {}
-
-  async poll(): Promise<StepTaskRecord<TFlow>[]> {
-    if (this.isAborted()) {
-      this.logger.debug('Polling aborted, returning empty array');
-      return [];
-    }
-
-    this.logger.debug(
-      `Polling for flow tasks with batch size ${this.config.batchSize}`
-    );
-
-    try {
-      const tasks = await this.adapter.pollForTasks(
-        this.config.queueName,
-        this.config.batchSize
-      );
-      this.logger.debug(`Retrieved ${tasks.length} flow tasks`);
-      return tasks;
-    } catch (err: unknown) {
-      this.logger.error(`Error polling for flow tasks: ${err}`);
-      return [];
-    }
-  }
-
-  private isAborted(): boolean {
-    return this.signal.aborted;
-  }
-}

package/src/flow/createFlowWorker.ts

@@ -1,105 +0,0 @@
-import type { AnyFlow } from '@pgflow/dsl';
-import type { EdgeWorkerConfig } from '../EdgeWorker.ts';
-import { ExecutionController } from '../core/ExecutionController.ts';
-import { StepTaskPoller, type StepTaskPollerConfig } from './StepTaskPoller.ts';
-import { StepTaskExecutor } from './StepTaskExecutor.ts';
-import { PgflowSqlClient } from '@pgflow/core';
-import { Queries } from '../core/Queries.ts';
-import type { StepTaskRecord } from './types.ts';
-import type { IExecutor } from '../core/types.ts';
-import { Worker } from '../core/Worker.ts';
-import postgres from 'postgres';
-import { FlowWorkerLifecycle } from './FlowWorkerLifecycle.ts';
-import { BatchProcessor } from '../core/BatchProcessor.ts';
-import { getLogger } from '../core/Logger.ts';
-
-/**
- * Configuration for the flow worker
- */
-export type FlowWorkerConfig = EdgeWorkerConfig & {
-  maxConcurrent?: number;
-  connectionString?: string;
-  sql?: postgres.Sql;
-  maxPgConnections?: number;
-  batchSize?: number;
-};
-
-/**
- * Creates a new Worker instance for processing flow tasks.
- *
- * @param flow - The Flow DSL definition
- * @param config - Configuration options for the worker
- * @returns A configured Worker instance ready to be started
- */
-export function createFlowWorker<TFlow extends AnyFlow>(
-  flow: TFlow,
-  config: FlowWorkerConfig
-): Worker {
-  const logger = getLogger('createFlowWorker');
-
-  // Create abort controller for graceful shutdown
-  const abortController = new AbortController();
-  const abortSignal = abortController.signal;
-
-  if (!config.sql && !config.connectionString) {
-    throw new Error(
-      "Either 'sql' or 'connectionString' must be provided in FlowWorkerConfig."
-    );
-  }
-
-  const sql =
-    config.sql ||
-    postgres(config.connectionString!, {
-      max: config.maxPgConnections,
-      prepare: false,
-    });
-
-  // Create the pgflow adapter
-  const pgflowAdapter = new PgflowSqlClient<TFlow>(sql);
-
-  // Use flow slug as queue name, or fallback to 'tasks'
-  const queueName = flow.slug || 'tasks';
-  logger.debug(`Using queue name: ${queueName}`);
-
-  // Create specialized FlowWorkerLifecycle with the proxied queue and flow
-  const queries = new Queries(sql);
-  const lifecycle = new FlowWorkerLifecycle<TFlow>(queries, flow);
-
-  // Create StepTaskPoller
-  const pollerConfig: StepTaskPollerConfig = {
-    batchSize: config.batchSize || 10,
-    queueName: flow.slug,
-  };
-  const poller = new StepTaskPoller<TFlow>(
-    pgflowAdapter,
-    abortSignal,
-    pollerConfig
-  );
-
-  // Create executor factory with proper typing
-  const executorFactory = (
-    record: StepTaskRecord<TFlow>,
-    signal: AbortSignal
-  ): IExecutor => {
-    return new StepTaskExecutor<TFlow>(flow, record, pgflowAdapter, signal);
-  };
-
-  // Create ExecutionController
-  const executionController = new ExecutionController<StepTaskRecord<TFlow>>(
-    executorFactory,
-    abortSignal,
-    {
-      maxConcurrent: config.maxConcurrent || 10,
-    }
-  );
-
-  // Create BatchProcessor
-  const batchProcessor = new BatchProcessor<StepTaskRecord<TFlow>>(
-    executionController,
-    poller,
-    abortSignal
-  );
-
-  // Return Worker
-  return new Worker(batchProcessor, lifecycle, sql);
-}

package/src/flow/types.ts

@@ -1 +0,0 @@
-export * from '../../../core/src/types.ts';

package/src/index.ts

@@ -1,15 +0,0 @@
-// Export existing queue-based worker
-export { createQueueWorker } from './queue/createQueueWorker.ts';
-export { EdgeWorker } from './EdgeWorker.ts';
-
-// Export new flow-based worker
-export { createFlowWorker } from './flow/createFlowWorker.ts';
-export { FlowWorkerLifecycle } from './flow/FlowWorkerLifecycle.ts';
-
-// Export types
-export type { StepTaskRecord } from './flow/types.ts';
-export type { FlowWorkerConfig } from './flow/createFlowWorker.ts';
-export type { StepTaskPollerConfig } from './flow/StepTaskPoller.ts';
-
-// Re-export types from the base system
-export type { Json, IExecutor, IPoller, IMessage, ILifecycle, IBatchProcessor } from './core/types.ts';

package/src/queue/MessageExecutor.ts

@@ -1,105 +0,0 @@
-import type { Json } from '../core/types.ts';
-import type { PgmqMessageRecord } from './types.ts';
-import type { Queue } from './Queue.ts';
-import { getLogger } from '../core/Logger.ts';
-
-class AbortError extends Error {
-  constructor() {
-    super('Operation aborted');
-    this.name = 'AbortError';
-  }
-}
-
-/**
- * A class that executes a message handler.
- *
- * It handles the execution of the message handler and retries or archives the message
- * based on the retry limit and delay.
- *
- * It also handles the abort signal and logs the error.
- */
-export class MessageExecutor<TPayload extends Json> {
-  private logger = getLogger('MessageExecutor');
-
-  constructor(
-    private readonly queue: Queue<TPayload>,
-    private readonly record: PgmqMessageRecord<TPayload>,
-    private readonly messageHandler: (
-      message: TPayload
-    ) => Promise<void> | void,
-    private readonly signal: AbortSignal,
-    private readonly retryLimit: number,
-    private readonly retryDelay: number
-  ) {}
-
-  get msgId() {
-    return this.record.msg_id;
-  }
-
-  async execute(): Promise<void> {
-    try {
-      if (this.signal.aborted) {
-        throw new AbortError();
-      }
-
-      // Check if already aborted before starting
-      this.signal.throwIfAborted();
-
-      this.logger.debug(`Executing task ${this.msgId}...`);
-      await this.messageHandler(this.record.message!);
-
-      this.logger.debug(
-        `Task ${this.msgId} completed successfully, archiving...`
-      );
-      await this.queue.archive(this.msgId);
-      this.logger.debug(`Archived task ${this.msgId} successfully`);
-    } catch (error) {
-      await this.handleExecutionError(error);
-    }
-  }
-
-  /**
-   * Handles the error that occurred during execution.
-   *
-   * If the error is an AbortError, it means that the worker was aborted and stopping,
-   * the message will reappear after the visibility timeout and be picked up by another worker.
-   *
-   * Otherwise, it proceeds with retry or archiving forever.
-   */
-  private async handleExecutionError(error: unknown) {
-    if (error instanceof Error && error.name === 'AbortError') {
-      this.logger.debug(`Aborted execution for ${this.msgId}`);
-      // Do not throw - the worker was aborted and stopping,
-      // the message will reappear after the visibility timeout
-      // and be picked up by another worker
-    } else {
-      this.logger.debug(`Task ${this.msgId} failed with error: ${error}`);
-      await this.retryOrArchive();
-    }
-  }
-
-  /**
-   * Retries the message if it is available.
-   * Otherwise, archives the message forever and stops processing it.
-   */
-  private async retryOrArchive() {
-    if (this.retryAvailable) {
-      // adjust visibility timeout for message to appear after retryDelay
-      this.logger.debug(`Retrying ${this.msgId} in ${this.retryDelay} seconds`);
-      await this.queue.setVt(this.msgId, this.retryDelay);
-    } else {
-      // archive message forever and stop processing it
-      this.logger.debug(`Archiving ${this.msgId} forever`);
-      await this.queue.archive(this.msgId);
-    }
-  }
-
-  /**
-   * Returns true if the message can be retried.
-   */
-  private get retryAvailable() {
-    const readCountLimit = this.retryLimit + 1; // initial read also counts
-
-    return this.record.read_ct < readCountLimit;
-  }
-}

package/src/queue/Queue.ts

@@ -1,92 +0,0 @@
-import type postgres from 'postgres';
-import type { PgmqMessageRecord } from './types.ts';
-import type { Json } from '../core/types.ts';
-
-export class Queue<TPayload extends Json> {
-  constructor(private readonly sql: postgres.Sql, readonly queueName: string) {}
-
-  /**
-   * Creates a queue if it doesn't exist.
-   * If the queue already exists, this method does nothing.
-   */
-  async safeCreate() {
-    return await this.sql`
-        select * from pgmq.create(${this.queueName})
-        where not exists (
-          select 1 from pgmq.list_queues() where queue_name = ${this.queueName}
-        );
-    `;
-  }
-
-  /**
-   * Drops a queue if it exists.
-   * If the queue doesn't exist, this method does nothing.
-   */
-  async safeDrop() {
-    return await this.sql`
-        select * from pgmq.drop_queue(${this.queueName})
-        where exists (
-          select 1 from pgmq.list_queues() where queue_name = ${this.queueName}
-        );
-    `;
-  }
-
-  async archive(msgId: number): Promise<void> {
-    await this.sql`
-      SELECT pgmq.archive(queue_name => ${this.queueName}, msg_id => ${msgId}::bigint);
-    `;
-  }
-
-  async archiveBatch(msgIds: number[]): Promise<void> {
-    await this.sql`
-      SELECT pgmq.archive(queue_name => ${this.queueName}, msg_ids => ${msgIds}::bigint[]);
-    `;
-  }
-
-  async send(message: TPayload): Promise<void> {
-    const msgJson = JSON.stringify(message);
-    await this.sql`
-      SELECT pgmq.send(queue_name => ${this.queueName}, msg => ${msgJson}::jsonb)
-    `;
-  }
-
-  async readWithPoll(
-    batchSize = 20,
-    visibilityTimeout = 2,
-    maxPollSeconds = 5,
-    pollIntervalMs = 200
-  ) {
-    return await this.sql<PgmqMessageRecord<TPayload>[]>`
-      SELECT *
-      FROM edge_worker.read_with_poll(
-        queue_name => ${this.queueName},
-        vt => ${visibilityTimeout},
-        qty => ${batchSize},
-        max_poll_seconds => ${maxPollSeconds},
-        poll_interval_ms => ${pollIntervalMs}
-      );
-    `;
-  }
-
-  /**
-   * Sets the visibility timeout of a message to the current time plus the given offset.
-   *
-   * This is an inlined version of the pgmq.set_vt in order to fix the bug.
-   * The original uses now() instead of clock_timestamp() which is problematic in transactions.
-   * See more details here: https://github.com/tembo-io/pgmq/issues/367
-   *
-   * The only change made is now() replaced with clock_timestamp().
-   */
-  async setVt(
-    msgId: number,
-    vtOffsetSeconds: number
-  ): Promise<PgmqMessageRecord<TPayload>> {
-    const records = await this.sql<PgmqMessageRecord<TPayload>[]>`
-      UPDATE ${this.sql('pgmq.q_' + this.queueName)}
-      SET vt = (clock_timestamp() + make_interval(secs => ${vtOffsetSeconds}))
-      WHERE msg_id = ${msgId}::bigint
-      RETURNING *;
-    `;
-    return records[0];
-  }
-}

package/src/queue/ReadWithPollPoller.ts

@@ -1,35 +0,0 @@
-import type { Queue } from './Queue.ts';
-import type { PgmqMessageRecord } from './types.ts';
-import type { Json } from '../core/types.ts';
-
-export interface PollerConfig {
-  batchSize: number;
-  maxPollSeconds: number;
-  pollIntervalMs: number;
-  visibilityTimeout: number;
-}
-
-export class ReadWithPollPoller<TPayload extends Json> {
-  constructor(
-    protected readonly queue: Queue<TPayload>,
-    protected readonly signal: AbortSignal,
-    protected readonly config: PollerConfig
-  ) {}
-
-  async poll(): Promise<PgmqMessageRecord<TPayload>[]> {
-    if (this.isAborted()) {
-      return [];
-    }
-
-    return await this.queue.readWithPoll(
-      this.config.batchSize,
-      this.config.visibilityTimeout,
-      this.config.maxPollSeconds,
-      this.config.pollIntervalMs
-    );
-  }
-
-  private isAborted(): boolean {
-    return this.signal.aborted;
-  }
-}

package/src/queue/createQueueWorker.ts

@@ -1,145 +0,0 @@
-import { ExecutionController } from '../core/ExecutionController.ts';
-import { MessageExecutor } from './MessageExecutor.ts';
-import { Queries } from '../core/Queries.ts';
-import { Queue } from './Queue.ts';
-import { ReadWithPollPoller } from './ReadWithPollPoller.ts';
-import type { Json } from '../core/types.ts';
-import type { PgmqMessageRecord } from './types.ts';
-import { Worker } from '../core/Worker.ts';
-import postgres from 'postgres';
-import { WorkerLifecycle } from '../core/WorkerLifecycle.ts';
-import { BatchProcessor } from '../core/BatchProcessor.ts';
-
-/**
- * Configuration for the queue worker
- */
-export type QueueWorkerConfig = {
-  /**
-   * PostgreSQL connection string.
-   * If not provided, it will be read from the EDGE_WORKER_DB_URL environment variable.
-   */
-  connectionString?: string;
-
-  /**
-   * Name of the queue to poll for messages
-   * @default 'tasks'
-   */
-  queueName?: string;
-
-  /**
-   * How many tasks are processed at the same time
-   * @default 10
-   */
-  maxConcurrent?: number;
-
-  /**
-   * How many connections to the database are opened
-   * @default 4
-   */
-  maxPgConnections?: number;
-
-  /**
-   * In-worker polling interval in seconds
-   * @default 5
-   */
-  maxPollSeconds?: number;
-
-  /**
-   * In-database polling interval in milliseconds
-   * @default 200
-   */
-  pollIntervalMs?: number;
-
-  /**
-   * How long to wait before retrying a failed job in seconds
-   * @default 5
-   */
-  retryDelay?: number;
-
-  /**
-   * How many times to retry a failed job
-   * @default 5
-   */
-  retryLimit?: number;
-
-  /**
-   * How long a job is invisible after reading in seconds.
-   * If not successful, will reappear after this time.
-   * @default 3
-   */
-  visibilityTimeout?: number;
-
-  /**
-   * Batch size for polling messages
-   * @default 10
-   */
-  batchSize?: number;
-
-  /**
-   * Optional SQL client instance
-   */
-  sql?: postgres.Sql;
-};
-
-/**
- * Creates a new Worker instance for processing queue messages.
- *
- * @param handler - The message handler function that processes each message from the queue
- * @param config - Configuration options for the worker
- * @returns A configured Worker instance ready to be started
- */
-export function createQueueWorker<TPayload extends Json>(
-  handler: (message: TPayload) => Promise<void> | void,
-  config: QueueWorkerConfig
-): Worker {
-  type QueueMessage = PgmqMessageRecord<TPayload>;
-
-  const abortController = new AbortController();
-  const abortSignal = abortController.signal;
-
-  // Use provided SQL connection if available, otherwise create one from connection string
-  const sql =
-    config.sql ||
-    postgres(config.connectionString || '', {
-      max: config.maxPgConnections,
-      prepare: false,
-    });
-
-  const queue = new Queue<TPayload>(sql, config.queueName || 'tasks');
-  const queries = new Queries(sql);
-
-  const lifecycle = new WorkerLifecycle<TPayload>(queries, queue);
-
-  const executorFactory = (record: QueueMessage, signal: AbortSignal) => {
-    return new MessageExecutor(
-      queue,
-      record,
-      handler,
-      signal,
-      config.retryLimit || 5,
-      config.retryDelay || 3
-    );
-  };
-
-  const poller = new ReadWithPollPoller(queue, abortSignal, {
-    batchSize: config.batchSize || config.maxConcurrent || 10,
-    maxPollSeconds: config.maxPollSeconds || 5,
-    pollIntervalMs: config.pollIntervalMs || 200,
-    visibilityTimeout: config.visibilityTimeout || 3,
-  });
-
-  const executionController = new ExecutionController<QueueMessage>(
-    executorFactory,
-    abortSignal,
-    {
-      maxConcurrent: config.maxConcurrent || 10,
-    }
-  );
-  const batchProcessor = new BatchProcessor<QueueMessage>(
-    executionController,
-    poller,
-    abortSignal
-  );
-
-  return new Worker(batchProcessor, lifecycle, sql);
-}

package/src/queue/types.ts

@@ -1,14 +0,0 @@
-import type { Json, IMessage } from "../core/types.ts";
-
-/**
- * Fields are nullable because types in postgres does not allow NOT NULL,
- * but all those values except `message` come from queue table columns,
- * which are explicitely marked as NOT NULL.
- */
-export interface PgmqMessageRecord<TPayload extends Json | null = Json> extends IMessage {
-  read_ct: number;
-  enqueued_at: string;
-  vt: string;
-  message: TPayload;
-}
-

package/src/spawnNewEdgeFunction.ts

@@ -1,33 +0,0 @@
-import { getLogger } from './core/Logger.ts';
-
-// @ts-ignore - TODO: fix the types
-const SUPABASE_URL = Deno.env.get('SUPABASE_URL') as string;
-const SUPABASE_ANON_KEY = Deno.env.get('SUPABASE_ANON_KEY') as string;
-
-const logger = getLogger('spawnNewEdgeFunction');
-
-export default async function spawnNewEdgeFunction(
-  functionName: string = 'edge-worker'
-): Promise<void> {
-  if (!functionName) {
-    throw new Error('functionName cannot be null or empty');
-  }
-
-  logger.debug('Spawning a new Edge Function...');
-
-  const response = await fetch(`${SUPABASE_URL}/functions/v1/${functionName}`, {
-    method: 'POST',
-    headers: {
-      Authorization: `Bearer ${SUPABASE_ANON_KEY}`,
-      'Content-Type': 'application/json',
-    }
-  });
-
-  logger.debug('Edge Function spawned successfully!');
-
-  if (!response.ok) {
-    throw new Error(
-      `Edge function returned non-OK status: ${response.status} ${response.statusText}`
-    );
-  }
-}

package/supabase/call

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# get env vars from supabase status and eval them for usage
-supabase_env=$(supabase status --output env 2>/dev/null)
-
-if [ $? -eq 0 ]; then
-  eval "$supabase_env"
-fi
-
-if [ -f ".env" ]; then
-  source .env
-fi
-
-function_name="$1"
-data="$2"
-
-API_URL=http://localhost:50321
-curl \
-  --request POST \
-  "${API_URL}/functions/v1/${function_name}" \
-  --header 'Authorization: Bearer '${ANON_KEY}'' \
-  --header 'Content-Type: application/json' \
-  --data-raw "$data"

package/supabase/cli

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-pnpm supabase --workdir pkgs/edge-worker $@

package/supabase/config.toml

@@ -1,42 +0,0 @@
-project_id = "edge-worker"
-
-[api]
-enabled = true
-port = 50321
-
-[db]
-port = 50322
-shadow_port = 50320
-major_version = 15
-
-[db.pooler]
-enabled = true
-port = 50329
-pool_mode = "transaction"
-default_pool_size = 200
-max_client_conn = 250
-
-[db.seed]
-enabled = true
-sql_paths = ['./seed.sql']
-
-[edge_runtime]
-enabled = true
-policy = "per_worker"
-inspector_port = 8083
-
-# disable unused features
-[realtime]
-enabled = false
-[studio]
-enabled = false
-[inbucket]
-enabled = false
-[analytics]
-enabled = false
-[storage]
-enabled = false
-[auth]
-enabled = false
-
-

package/supabase/functions/cpu_intensive/index.ts

@@ -1,20 +0,0 @@
-import { EdgeWorker } from '../_src/EdgeWorker.ts';
-import { crypto } from 'jsr:@std/crypto';
-import { sql } from '../utils.ts';
-
-async function cpuIntensiveTask() {
-  let data = new TextEncoder().encode('burn');
-  const timeId = `cpu_intensive_${Math.random()}`;
-  console.time(timeId);
-  for (let i = 0; i < 10000; i++) {
-    data = new Uint8Array(await crypto.subtle.digest('SHA-256', data));
-  }
-  console.timeEnd(timeId);
-
-  console.log(
-    '[cpu_intensive] last_val = ',
-    await sql`SELECT nextval('test_seq')`
-  );
-}
-
-EdgeWorker.start(cpuIntensiveTask, { queueName: 'cpu_intensive' });