@pgflow/edge-worker 0.0.5 → 0.0.7-prealpha.1

package/src/core/BatchProcessor.ts

@@ -1,38 +0,0 @@
-import type { ExecutionController } from './ExecutionController.ts';
-import type { IMessage, IPoller } from './types.ts';
-import { getLogger } from './Logger.ts';
-
-export class BatchProcessor<TMessage extends IMessage> {
-  private logger = getLogger('BatchProcessor');
-
-  constructor(
-    private executionController: ExecutionController<TMessage>,
-    private poller: IPoller<TMessage>,
-    private signal: AbortSignal
-  ) {
-    this.executionController = executionController;
-    this.signal = signal;
-    this.poller = poller;
-  }
-
-  async processBatch() {
-    this.logger.debug('Polling for new batch of messages...');
-    const messageRecords = await this.poller.poll();
-
-    if (this.signal.aborted) {
-      this.logger.info('Discarding messageRecords because worker is stopping');
-      return;
-    }
-
-    this.logger.debug(`Starting ${messageRecords.length} messages`);
-
-    const startPromises = messageRecords.map(
-      (message) => this.executionController.start(message)
-    );
-    await Promise.all(startPromises);
-  }
-
-  async awaitCompletion() {
-    return await this.executionController.awaitCompletion();
-  }
-}

package/src/core/ExecutionController.ts

@@ -1,51 +0,0 @@
-import { newQueue, type Queue as PromiseQueue } from '@henrygd/queue';
-import type { IExecutor, IMessage } from './types.ts';
-import { getLogger } from './Logger.ts';
-
-export interface ExecutionConfig {
-  maxConcurrent: number;
-}
-
-export class ExecutionController<TMessage extends IMessage> {
-  private logger = getLogger('ExecutionController');
-  private promiseQueue: PromiseQueue;
-  private signal: AbortSignal;
-  private createExecutor: (record: TMessage, signal: AbortSignal) => IExecutor;
-
-  constructor(
-    executorFactory: (record: TMessage, signal: AbortSignal) => IExecutor,
-    abortSignal: AbortSignal,
-    config: ExecutionConfig
-  ) {
-    this.signal = abortSignal;
-    this.createExecutor = executorFactory;
-    this.promiseQueue = newQueue(config.maxConcurrent);
-  }
-
-  async start(record: TMessage) {
-    const executor = this.createExecutor(record, this.signal);
-
-    this.logger.info(`Scheduling execution of task ${executor.msgId}`);
-
-    return await this.promiseQueue.add(async () => {
-      try {
-        this.logger.debug(`Executing task ${executor.msgId}...`);
-        await executor.execute();
-        this.logger.debug(`Execution successful for ${executor.msgId}`);
-      } catch (error) {
-        this.logger.error(`Execution failed for ${executor.msgId}:`, error);
-        throw error;
-      }
-    });
-  }
-
-  async awaitCompletion() {
-    const active = this.promiseQueue.active();
-    const all = this.promiseQueue.size();
-
-    this.logger.debug(
-      `Awaiting completion of all tasks... (active/all: ${active}}/${all})`
-    );
-    await this.promiseQueue.done();
-  }
-}

package/src/core/Heartbeat.ts

@@ -1,23 +0,0 @@
-import type { Queries } from './Queries.ts';
-import type { WorkerRow } from './types.ts';
-import { getLogger } from './Logger.ts';
-
-export class Heartbeat {
-  private logger = getLogger('Heartbeat');
-  private lastHeartbeat = 0;
-
-  constructor(
-    private interval: number,
-    private queries: Queries,
-    private workerRow: WorkerRow
-  ) {}
-
-  async send(): Promise<void> {
-    const now = Date.now();
-    if (now - this.lastHeartbeat >= this.interval) {
-      await this.queries.sendHeartbeat(this.workerRow);
-      this.logger.debug('OK');
-      this.lastHeartbeat = now;
-    }
-  }
-}

package/src/core/Logger.ts

@@ -1,42 +0,0 @@
-import pino from 'pino';
-
-function getLogLevelFromEnv(): pino.LevelWithSilent {
-  const validLevels = [
-    'DEBUG',
-    'INFO',
-    'ERROR',
-  ];
-  const logLevel = Deno.env.get('EDGE_WORKER_LOG_LEVEL')?.toUpperCase();
-
-  if (logLevel && !validLevels.includes(logLevel)) {
-    console.warn(`Invalid log level "${logLevel}". Using "INFO" instead.`);
-    return 'info';
-  }
-
-  return (logLevel?.toLowerCase() as pino.LevelWithSilent) || 'info';
-}
-
-export function setupLogger(workerId: string) {
-  const level = getLogLevelFromEnv();
-  
-  const loggerOptions: pino.LoggerOptions = {
-    level,
-    formatters: {
-      bindings: () => ({ worker_id: workerId }),
-    },
-    serializers: pino.stdSerializers,
-    transport: {
-      target: 'pino-pretty',
-      options: {
-        colorize: true,
-        messageFormat: '[{module}] {msg}',
-      }
-    }
-  };
-
-  pino(loggerOptions);
-}
-
-export function getLogger(module: string) {
-  return pino().child({ module });
-}

package/src/core/Queries.ts

@@ -1,44 +0,0 @@
-import type postgres from 'postgres';
-import type { WorkerRow } from './types.ts';
-
-export class Queries {
-  constructor(private readonly sql: postgres.Sql) {}
-
-  async onWorkerStarted({
-    queueName,
-    workerId,
-    edgeFunctionName,
-  }: {
-    queueName: string;
-    workerId: string;
-    edgeFunctionName: string;
-  }): Promise<WorkerRow> {
-    const [worker] = await this.sql<WorkerRow[]>`
-      INSERT INTO edge_worker.workers (queue_name, worker_id, function_name)
-      VALUES (${queueName}, ${workerId}, ${edgeFunctionName})
-      RETURNING *;
-    `;
-
-    return worker;
-  }
-
-  async onWorkerStopped(workerRow: WorkerRow): Promise<WorkerRow> {
-    const [worker] = await this.sql<WorkerRow[]>`
-      UPDATE edge_worker.workers AS w
-      SET stopped_at = clock_timestamp(), last_heartbeat_at = clock_timestamp()
-      WHERE w.worker_id = ${workerRow.worker_id}
-      RETURNING *;
-    `;
-
-    return worker;
-  }
-
-  async sendHeartbeat(workerRow: WorkerRow): Promise<void> {
-    await this.sql<WorkerRow[]>`
-      UPDATE edge_worker.workers AS w
-      SET last_heartbeat_at = clock_timestamp()
-      WHERE w.worker_id = ${workerRow.worker_id}
-      RETURNING *;
-    `;
-  }
-}

package/src/core/Worker.ts

@@ -1,102 +0,0 @@
-import type postgres from 'postgres';
-import type { IBatchProcessor, ILifecycle, WorkerBootstrap } from './types.ts';
-import { getLogger, setupLogger } from './Logger.ts';
-
-export class Worker {
-  private lifecycle: ILifecycle;
-  private logger = getLogger('Worker');
-  private abortController = new AbortController();
-
-  private batchProcessor: IBatchProcessor;
-  private sql: postgres.Sql;
-
-  constructor(
-    batchProcessor: IBatchProcessor,
-    lifecycle: ILifecycle,
-    sql: postgres.Sql,
-  ) {
-    this.sql = sql;
-
-    this.lifecycle = lifecycle;
-
-    this.batchProcessor = batchProcessor;
-  }
-
-  async startOnlyOnce(workerBootstrap: WorkerBootstrap) {
-    if (this.lifecycle.isRunning) {
-      this.logger.debug('Worker already running, ignoring start request');
-      return;
-    }
-
-    await this.start(workerBootstrap);
-  }
-
-  private async start(workerBootstrap: WorkerBootstrap) {
-    setupLogger(workerBootstrap.workerId);
-
-    try {
-      await this.lifecycle.acknowledgeStart(workerBootstrap);
-
-      while (this.isMainLoopActive) {
-        try {
-          await this.lifecycle.sendHeartbeat();
-        } catch (error: unknown) {
-          this.logger.error(`Error sending heartbeat: ${error}`);
-          // Continue execution - a failed heartbeat shouldn't stop processing
-        }
-
-        try {
-          await this.batchProcessor.processBatch();
-        } catch (error: unknown) {
-          this.logger.error(`Error processing batch: ${error}`);
-          // Continue to next iteration - failed batch shouldn't stop the worker
-        }
-      }
-    } catch (error) {
-      this.logger.error(`Error in worker main loop: ${error}`);
-      throw error;
-    }
-  }
-
-  async stop() {
-    // If the worker is already stopping or stopped, do nothing
-    if (this.lifecycle.isStopping || this.lifecycle.isStopped) {
-      return;
-    }
-
-    this.lifecycle.transitionToStopping();
-
-    try {
-      this.logger.info('-> Stopped accepting new messages');
-      this.abortController.abort();
-
-      this.logger.info('-> Waiting for pending tasks to complete...');
-      await this.batchProcessor.awaitCompletion();
-      this.logger.info('-> Pending tasks completed!');
-
-      this.lifecycle.acknowledgeStop();
-
-      this.logger.info('-> Closing SQL connection...');
-      await this.sql.end();
-      this.logger.info('-> SQL connection closed!');
-    } catch (error) {
-      this.logger.info(`Error during worker stop: ${error}`);
-      throw error;
-    }
-  }
-
-  get edgeFunctionName() {
-    return this.lifecycle.edgeFunctionName;
-  }
-
-  /**
-   * Returns true if worker state is Running and worker was not stopped
-   */
-  private get isMainLoopActive() {
-    return this.lifecycle.isRunning && !this.isAborted;
-  }
-
-  private get isAborted() {
-    return this.abortController.signal.aborted;
-  }
-}

package/src/core/WorkerLifecycle.ts

@@ -1,93 +0,0 @@
-import { Heartbeat } from './Heartbeat.ts';
-import { getLogger } from './Logger.ts';
-import type { Queries } from './Queries.ts';
-import type { Queue } from '../queue/Queue.ts';
-import type { ILifecycle, Json, WorkerBootstrap, WorkerRow } from './types.ts';
-import { States, WorkerState } from './WorkerState.ts';
-
-export interface LifecycleConfig {
-  queueName: string;
-}
-
-export class WorkerLifecycle<IMessage extends Json> implements ILifecycle {
-  private workerState: WorkerState = new WorkerState();
-  private heartbeat?: Heartbeat;
-  private logger = getLogger('WorkerLifecycle');
-  private queries: Queries;
-  private queue: Queue<IMessage>;
-  private workerRow?: WorkerRow;
-
-  constructor(queries: Queries, queue: Queue<IMessage>) {
-    this.queries = queries;
-    this.queue = queue;
-  }
-
-  async acknowledgeStart(workerBootstrap: WorkerBootstrap): Promise<void> {
-    this.workerState.transitionTo(States.Starting);
-
-    this.logger.info(`Ensuring queue '${this.queue.queueName}' exists...`);
-    await this.queue.safeCreate();
-
-    this.workerRow = await this.queries.onWorkerStarted({
-      queueName: this.queueName,
-      ...workerBootstrap,
-    });
-
-    this.heartbeat = new Heartbeat(5000, this.queries, this.workerRow);
-
-    this.workerState.transitionTo(States.Running);
-  }
-
-  acknowledgeStop() {
-    this.workerState.transitionTo(States.Stopping);
-
-    if (!this.workerRow) {
-      throw new Error('Cannot stop worker: workerRow not set');
-    }
-
-    try {
-      this.logger.debug('Acknowledging worker stop...');
-
-      // TODO: commented out because we can live without this
-      //       but it is causing problems with DbHandler - workes does not have
-      //       enough time to fire this query before hard-terimnated
-      //       We can always check the heartbeat to see if it is still running
-      //
-      // await this.queries.onWorkerStopped(this.workerRow);
-
-      this.workerState.transitionTo(States.Stopped);
-      this.logger.debug('Worker stop acknowledged');
-    } catch (error) {
-      this.logger.debug(`Error acknowledging worker stop: ${error}`);
-      throw error;
-    }
-  }
-
-  get edgeFunctionName() {
-    return this.workerRow?.function_name;
-  }
-
-  get queueName() {
-    return this.queue.queueName;
-  }
-
-  async sendHeartbeat() {
-    await this.heartbeat?.send();
-  }
-
-  get isRunning() {
-    return this.workerState.isRunning;
-  }
-
-  get isStopping() {
-    return this.workerState.isStopping;
-  }
-
-  get isStopped() {
-    return this.workerState.isStopped;
-  }
-
-  transitionToStopping() {
-    this.workerState.transitionTo(States.Stopping);
-  }
-}

package/src/core/WorkerState.ts

@@ -1,85 +0,0 @@
-import { getLogger } from './Logger.ts';
-
-export enum States {
-  /** The worker has been created but has not yet started. */
-  Created = 'created',
-
-  /** The worker is starting but has not yet started processing messages. */
-  Starting = 'starting',
-
-  /** The worker is processing messages. */
-  Running = 'running',
-
-  /** The worker stopped processing messages but is still releasing resources. */
-  Stopping = 'stopping',
-
-  /** The worker has stopped processing messages and released resources
-   * and can be discarded. */
-  Stopped = 'stopped',
-}
-
-export const Transitions: Record<States, States[]> = {
-  [States.Created]: [States.Starting],
-  [States.Starting]: [States.Running],
-  [States.Running]: [States.Stopping],
-  [States.Stopping]: [States.Stopped],
-  [States.Stopped]: [], // Terminal state - no valid transitions from here
-};
-
-export class TransitionError extends Error {
-  constructor(options: { from: States; to: States }) {
-    super(`Cannot transition from ${options.from} to ${options.to}`);
-  }
-}
-
-/**
- * Represents the state of a worker and exposes method for doing allowed transitions
- */
-export class WorkerState {
-  private logger = getLogger('WorkerState');
-  private state: States = States.Created;
-
-  get current() {
-    return this.state;
-  }
-
-  get isCreated() {
-    return this.state === States.Created;
-  }
-
-  get isStarting() {
-    return this.state === States.Starting;
-  }
-
-  get isRunning() {
-    return this.state === States.Running;
-  }
-
-  get isStopping() {
-    return this.state === States.Stopping;
-  }
-
-  get isStopped() {
-    return this.state === States.Stopped;
-  }
-
-  transitionTo(state: States) {
-    this.logger.debug(
-      `[WorkerState] Starting transition to '${state}' (current state: ${this.state})`
-    );
-
-    if (this.state === state) {
-      return;
-    }
-
-    if (Transitions[this.state].includes(state)) {
-      this.state = state;
-      this.logger.debug(`[WorkerState] Transitioned to '${state}'`);
-    } else {
-      throw new TransitionError({
-        from: this.state,
-        to: state,
-      });
-    }
-  }
-}

package/src/core/types.ts

@@ -1,47 +0,0 @@
-export type { Json } from '../../../core/src/types.ts';
-
-export interface IPoller<IMessage> {
-  poll(): Promise<IMessage[]>;
-}
-
-export interface IExecutor {
-  get msgId(): number;
-  execute(): Promise<unknown>;
-}
-
-export interface IMessage {
-  msg_id: number;
-}
-
-export interface ILifecycle {
-  acknowledgeStart(workerBootstrap: WorkerBootstrap): Promise<void>;
-  acknowledgeStop(): void;
-  sendHeartbeat(): Promise<void>;
-
-  get edgeFunctionName(): string | undefined;
-  get queueName(): string;
-  get isRunning(): boolean;
-  get isStopping(): boolean;
-  get isStopped(): boolean;
-
-  transitionToStopping(): void;
-}
-
-export interface IBatchProcessor {
-  processBatch(): Promise<void>;
-  awaitCompletion(): Promise<void>;
-}
-
-export type WorkerRow = {
-  last_heartbeat_at: string;
-  queue_name: string;
-  started_at: string;
-  stopped_at: string | null;
-  worker_id: string;
-  function_name: string;
-};
-
-export interface WorkerBootstrap {
-  edgeFunctionName: string;
-  workerId: string;
-}

package/src/flow/FlowWorkerLifecycle.ts

@@ -1,81 +0,0 @@
-import { Heartbeat } from '../core/Heartbeat.ts';
-import { getLogger } from '../core/Logger.ts';
-import type { Queries } from '../core/Queries.ts';
-import type { ILifecycle, WorkerBootstrap, WorkerRow } from '../core/types.ts';
-import { States, WorkerState } from '../core/WorkerState.ts';
-import type { AnyFlow } from '@pgflow/dsl';
-
-/**
- * A specialized WorkerLifecycle for Flow-based workers that is aware of the Flow's step types
- */
-export class FlowWorkerLifecycle<TFlow extends AnyFlow> implements ILifecycle {
-  private workerState: WorkerState = new WorkerState();
-  private heartbeat?: Heartbeat;
-  private logger = getLogger('FlowWorkerLifecycle');
-  private queries: Queries;
-  private workerRow?: WorkerRow;
-  private flow: TFlow;
-
-  constructor(queries: Queries, flow: TFlow) {
-    this.queries = queries;
-    this.flow = flow;
-  }
-
-  async acknowledgeStart(workerBootstrap: WorkerBootstrap): Promise<void> {
-    this.workerState.transitionTo(States.Starting);
-
-    this.workerRow = await this.queries.onWorkerStarted({
-      queueName: this.queueName,
-      ...workerBootstrap,
-    });
-
-    this.heartbeat = new Heartbeat(5000, this.queries, this.workerRow);
-
-    this.workerState.transitionTo(States.Running);
-  }
-
-  acknowledgeStop() {
-    this.workerState.transitionTo(States.Stopping);
-
-    if (!this.workerRow) {
-      throw new Error('Cannot stop worker: workerRow not set');
-    }
-
-    try {
-      this.logger.debug('Acknowledging worker stop...');
-      this.workerState.transitionTo(States.Stopped);
-      this.logger.debug('Worker stop acknowledged');
-    } catch (error) {
-      this.logger.debug(`Error acknowledging worker stop: ${error}`);
-      throw error;
-    }
-  }
-
-  get edgeFunctionName() {
-    return this.workerRow?.function_name;
-  }
-
-  get queueName() {
-    return this.flow.slug;
-  }
-
-  async sendHeartbeat() {
-    await this.heartbeat?.send();
-  }
-
-  get isRunning() {
-    return this.workerState.isRunning;
-  }
-
-  get isStopping() {
-    return this.workerState.isStopping;
-  }
-
-  get isStopped() {
-    return this.workerState.isStopped;
-  }
-
-  transitionToStopping() {
-    this.workerState.transitionTo(States.Stopping);
-  }
-}

package/src/flow/StepTaskExecutor.ts

@@ -1,87 +0,0 @@
-import type { AnyFlow } from '@pgflow/dsl';
-import type { StepTaskRecord, IPgflowClient } from './types.ts';
-import type { IExecutor } from '../core/types.ts';
-import { getLogger } from '../core/Logger.ts';
-
-class AbortError extends Error {
-  constructor() {
-    super('Operation aborted');
-    this.name = 'AbortError';
-  }
-}
-
-/**
- * An executor that processes step tasks using an IPgflowClient
- * with strong typing for the flow's step handlers
- */
-export class StepTaskExecutor<TFlow extends AnyFlow> implements IExecutor {
-  private logger = getLogger('StepTaskExecutor');
-
-  constructor(
-    private readonly flow: TFlow,
-    private readonly task: StepTaskRecord<TFlow>,
-    private readonly adapter: IPgflowClient<TFlow>,
-    private readonly signal: AbortSignal
-  ) {}
-
-  get msgId() {
-    return this.task.msg_id;
-  }
-
-  async execute(): Promise<void> {
-    try {
-      if (this.signal.aborted) {
-        throw new AbortError();
-      }
-
-      // Check if already aborted before starting
-      this.signal.throwIfAborted();
-
-      const stepSlug = this.task.step_slug;
-      this.logger.debug(
-        `Executing step task ${this.task.msg_id} for step ${stepSlug}`
-      );
-
-      // Get the step handler from the flow with proper typing
-      const stepDef = this.flow.getStepDefinition(stepSlug);
-
-      if (!stepDef) {
-        throw new Error(`No step definition found for slug=${stepSlug}`);
-      }
-
-      // !!! HANDLER EXECUTION !!!
-      const result = await stepDef.handler(this.task.input);
-      // !!! HANDLER EXECUTION !!!
-
-      this.logger.debug(
-        `step task ${this.task.msg_id} completed successfully, marking as complete`
-      );
-      await this.adapter.completeTask(this.task, result);
-
-      this.logger.debug(`step task ${this.task.msg_id} marked as complete`);
-    } catch (error) {
-      await this.handleExecutionError(error);
-    }
-  }
-
-  /**
-   * Handles the error that occurred during execution.
-   *
-   * If the error is an AbortError, it means that the worker was aborted and stopping,
-   * the task will be picked up by another worker later.
-   *
-   * Otherwise, it marks the task as failed.
-   */
-  private async handleExecutionError(error: unknown) {
-    if (error instanceof Error && error.name === 'AbortError') {
-      this.logger.debug(`Aborted execution for step task ${this.task.msg_id}`);
-      // Do not mark as failed - the worker was aborted and stopping,
-      // the task will be picked up by another worker later
-    } else {
-      this.logger.error(
-        `step task ${this.task.msg_id} failed with error: ${error}`
-      );
-      await this.adapter.failTask(this.task, error);
-    }
-  }
-}