flowfn 0.0.1

package/src/adapters/base.ts

@@ -0,0 +1,46 @@
+import { Job, QueueStats } from "../queue/types.js";
+import {
+  Message,
+  MessageHandler,
+  Subscription,
+  StreamInfo,
+} from "../stream/types.js";
+import { WorkflowExecution } from "../workflow/types.js";
+
+export interface FlowAdapter {
+  // Queue operations
+  enqueue(queue: string, job: Job): Promise<string>;
+  dequeue(queue: string): Promise<Job | null>;
+  ack(queue: string, jobId: string): Promise<void>;
+  nack(queue: string, jobId: string, requeue?: boolean): Promise<void>;
+
+  // Stream operations
+  publish(stream: string, message: Message): Promise<string>;
+  subscribe(
+    stream: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription>;
+  consume(
+    stream: string,
+    group: string,
+    consumer: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription>;
+  createConsumerGroup(stream: string, group: string): Promise<void>;
+
+  // Workflow operations
+  saveWorkflowState(
+    workflowId: string,
+    state: WorkflowExecution
+  ): Promise<void>;
+  loadWorkflowState(workflowId: string): Promise<WorkflowExecution | null>;
+
+  // Management
+  getJob(queue: string, jobId: string): Promise<Job | null>;
+  getJobs(queue: string, status: string): Promise<Job[]>;
+  getAllJobs(queue: string): Promise<Job[]>;
+  cleanJobs(queue: string, grace: number, status: string): Promise<number>;
+  getQueueStats(queue: string): Promise<QueueStats>;
+  getStreamInfo(stream: string): Promise<StreamInfo>;
+  cleanup(): Promise<void>;
+}

package/src/adapters/memory.ts

@@ -0,0 +1,183 @@
+import { FlowAdapter } from "./base.js";
+import { Job, QueueStats } from "../queue/types.js";
+import {
+  Message,
+  MessageHandler,
+  Subscription,
+  StreamInfo,
+} from "../stream/types.js";
+import { WorkflowExecution } from "../workflow/types.js";
+import { EventEmitter } from "eventemitter3";
+
+export class MemoryAdapter implements FlowAdapter {
+  private queues: Map<string, Job[]> = new Map();
+  private allJobs: Map<string, Job> = new Map();
+  private streams: Map<string, Message[]> = new Map();
+  private streamEmitters: Map<string, EventEmitter> = new Map();
+  private workflowStates: Map<string, WorkflowExecution> = new Map();
+
+  async enqueue(queueName: string, job: Job): Promise<string> {
+    if (!this.queues.has(queueName)) {
+      this.queues.set(queueName, []);
+    }
+    const queue = this.queues.get(queueName)!;
+    queue.push(job);
+    this.allJobs.set(job.id, job);
+    return job.id;
+  }
+
+  async dequeue(queueName: string): Promise<Job | null> {
+    const queue = this.queues.get(queueName);
+    if (!queue || queue.length === 0) return null;
+    return queue.shift() || null;
+  }
+
+  async ack(queue: string, jobId: string): Promise<void> {
+    const job = this.allJobs.get(jobId);
+    if (job) {
+      job.state = "completed";
+      job.finishedOn = Date.now();
+    }
+  }
+
+  async nack(
+    queueName: string,
+    jobId: string,
+    requeue: boolean = true
+  ): Promise<void> {
+    const job = this.allJobs.get(jobId);
+    if (job && requeue) {
+      job.state = "waiting";
+      const queue = this.queues.get(queueName);
+      if (queue) queue.push(job);
+    }
+  }
+
+  async getJob(queueName: string, jobId: string): Promise<Job | null> {
+    return this.allJobs.get(jobId) || null;
+  }
+
+  async getJobs(queue: string, status: string): Promise<Job[]> {
+    return Array.from(this.allJobs.values()).filter(
+      (job) => job.state === status
+    );
+  }
+
+  async getAllJobs(queue: string): Promise<Job[]> {
+    return Array.from(this.allJobs.values());
+  }
+
+  async cleanJobs(
+    queue: string,
+    grace: number,
+    status: string
+  ): Promise<number> {
+    const now = Date.now();
+    const jobsToClean: string[] = [];
+
+    for (const [id, job] of this.allJobs.entries()) {
+      if (job.state === status) {
+        const timestamp = job.finishedOn || job.timestamp;
+        if (now - timestamp > grace) {
+          jobsToClean.push(id);
+        }
+      }
+    }
+
+    for (const id of jobsToClean) {
+      this.allJobs.delete(id);
+    }
+
+    return jobsToClean.length;
+  }
+
+  async publish(streamName: string, message: Message): Promise<string> {
+    if (!this.streams.has(streamName)) {
+      this.streams.set(streamName, []);
+      this.streamEmitters.set(streamName, new EventEmitter());
+    }
+    const stream = this.streams.get(streamName)!;
+    stream.push(message);
+    this.streamEmitters.get(streamName)!.emit("message", message);
+    return message.id;
+  }
+
+  async subscribe(
+    streamName: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription> {
+    if (!this.streamEmitters.has(streamName)) {
+      this.streamEmitters.set(streamName, new EventEmitter());
+      this.streams.set(streamName, []);
+    }
+    const emitter = this.streamEmitters.get(streamName)!;
+    const wrapper = (msg: Message) => {
+      handler(msg).catch((err) => console.error("Stream handler error", err));
+    };
+    emitter.on("message", wrapper);
+
+    return {
+      unsubscribe: async () => {
+        emitter.off("message", wrapper);
+      },
+    };
+  }
+
+  async consume(
+    stream: string,
+    group: string,
+    consumer: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription> {
+    return this.subscribe(stream, handler);
+  }
+
+  async createConsumerGroup(stream: string, group: string): Promise<void> {
+    // No-op for memory
+  }
+
+  async saveWorkflowState(
+    workflowId: string,
+    state: WorkflowExecution
+  ): Promise<void> {
+    this.workflowStates.set(workflowId, state);
+  }
+
+  async loadWorkflowState(
+    workflowId: string
+  ): Promise<WorkflowExecution | null> {
+    return this.workflowStates.get(workflowId) || null;
+  }
+
+  async getQueueStats(queueName: string): Promise<QueueStats> {
+    const length = this.queues.get(queueName)?.length || 0;
+    const allInQueue = Array.from(this.allJobs.values()).filter(
+      (j) => j.state === "completed"
+    ).length;
+    return {
+      waiting: length,
+      active: 0,
+      completed: allInQueue,
+      failed: 0,
+      delayed: 0,
+      paused: 0,
+    };
+  }
+
+  async getStreamInfo(streamName: string): Promise<StreamInfo> {
+    const length = this.streams.get(streamName)?.length || 0;
+    return {
+      name: streamName,
+      length,
+      groups: 0,
+    };
+  }
+
+  async cleanup(): Promise<void> {
+    this.queues.clear();
+    this.allJobs.clear();
+    this.streams.clear();
+    this.workflowStates.clear();
+    this.streamEmitters.clear();
+  }
+}

package/src/adapters/postgres/index.ts

@@ -0,0 +1,383 @@
+import { FlowAdapter } from "../base.js";
+import { Job, QueueStats } from "../../queue/types.js";
+import {
+  Message,
+  MessageHandler,
+  Subscription,
+  StreamInfo,
+} from "../../stream/types.js";
+import { WorkflowExecution } from "../../workflow/types.js";
+import {
+  jobs,
+  messages,
+  workflowExecutions,
+  consumerGroups,
+} from "./schema.js";
+import { eq, and, lt, asc, sql } from "drizzle-orm";
+import { v4 as uuidv4 } from "uuid";
+
+export interface PostgresAdapterOptions {
+  db: any; // Drizzle instance
+  schema?: string;
+  pollInterval?: number;
+}
+
+export class PostgresAdapter implements FlowAdapter {
+  private db: any;
+  private pollInterval: number;
+  private activeSubscriptions: Map<string, boolean> = new Map();
+
+  constructor(options: PostgresAdapterOptions) {
+    this.db = options.db;
+    this.pollInterval = options.pollInterval || 1000;
+  }
+
+  async enqueue(queueName: string, job: Job): Promise<string> {
+    await this.db.insert(jobs).values({
+      id: job.id,
+      queue: queueName,
+      name: job.name,
+      data: job.data,
+      opts: job.opts,
+      state: job.opts.delay ? "delayed" : "waiting",
+      timestamp: job.timestamp,
+      delay: job.opts.delay || 0,
+      priority: job.opts.priority || 0,
+      attemptsMade: 0,
+    });
+    return job.id;
+  }
+
+  async dequeue(queueName: string): Promise<Job | null> {
+    // Attempt SKIP LOCKED if supported (Postgres)
+    // Drizzle doesn't have a standardized "for update skip locked" across all dialects yet in generic query builder,
+    // but assuming Postgres dialect here as per name.
+
+    // We want to find a job that is 'waiting' OR ('delayed' AND timestamp + delay <= now)
+    // And mark it 'active' atomically.
+
+    const now = Date.now();
+
+    return await this.db.transaction(async (tx: any) => {
+      // This is a raw SQL approach for Postgres SKIP LOCKED as Drizzle query builder support varies
+      // Assuming 'jobs' table is 'flowfn_jobs'
+
+      // 1. Check for delayed jobs that are ready
+      // (In a real high-perf system, a background process might move delayed -> waiting)
+      // For simplicity, we check both here.
+
+      const result = await tx.execute(sql`
+            UPDATE flowfn_jobs
+            SET state = 'active', processed_on = ${now}
+            WHERE id = (
+                SELECT id
+                FROM flowfn_jobs
+                WHERE queue = ${queueName}
+                AND (
+                    state = 'waiting'
+                    OR (state = 'delayed' AND (timestamp + delay) <= ${now})
+                )
+                ORDER BY priority DESC, timestamp ASC
+                FOR UPDATE SKIP LOCKED
+                LIMIT 1
+            )
+            RETURNING *
+        `);
+
+      if (result.length === 0) return null;
+
+      const row = result[0];
+      // Normalize back to Job object
+      return {
+        id: row.id,
+        name: row.name,
+        data: row.data,
+        opts: row.opts,
+        state: row.state,
+        progress: row.progress,
+        returnvalue: row.return_value,
+        timestamp: Number(row.timestamp),
+        processedOn: Number(row.processed_on),
+        finishedOn: Number(row.finished_on),
+        delay: Number(row.delay),
+        attemptsMade: row.attempts_made,
+        failedReason: row.failed_reason,
+        stacktrace: row.stacktrace,
+        // Re-bind methods in queue implementation
+      } as unknown as Job;
+    });
+  }
+
+  async ack(queue: string, jobId: string): Promise<void> {
+    // Usually mark as completed or delete.
+    // If removeOnComplete, delete.
+    // For now, mark completed.
+    await this.db
+      .update(jobs)
+      .set({ state: "completed", finishedOn: Date.now() })
+      .where(eq(jobs.id, jobId));
+  }
+
+  async nack(
+    queue: string,
+    jobId: string,
+    requeue: boolean = true
+  ): Promise<void> {
+    if (requeue) {
+      await this.db
+        .update(jobs)
+        .set({ state: "waiting", processedOn: null }) // Reset for retry
+        .where(eq(jobs.id, jobId));
+    } else {
+      // Leave as active? Or failed?
+    }
+  }
+
+  async publish(streamName: string, message: Message): Promise<string> {
+    await this.db.insert(messages).values({
+      id: message.id,
+      stream: streamName,
+      data: message.data,
+      headers: message.headers,
+      timestamp: message.timestamp,
+      partition: message.partition,
+      key: message.key,
+    });
+    return message.id;
+  }
+
+  async subscribe(
+    streamName: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription> {
+    // Polling implementation for streams
+    // Real postgres could use LISTEN/NOTIFY
+
+    const subId = uuidv4();
+    this.activeSubscriptions.set(subId, true);
+
+    let lastTimestamp = Date.now(); // Start from now
+
+    const poll = async () => {
+      if (!this.activeSubscriptions.get(subId)) return;
+
+      try {
+        const msgs = await this.db
+          .select()
+          .from(messages)
+          .where(
+            and(
+              eq(messages.stream, streamName),
+              sql`${messages.timestamp} > ${lastTimestamp}`
+            )
+          )
+          .orderBy(asc(messages.timestamp))
+          .limit(100);
+
+        for (const row of msgs) {
+          lastTimestamp = Math.max(lastTimestamp, Number(row.timestamp));
+
+          const msg: Message = {
+            id: row.id,
+            stream: row.stream,
+            data: row.data,
+            headers: row.headers as any,
+            timestamp: Number(row.timestamp),
+            ack: async () => {},
+            nack: async () => {},
+          };
+          handler(msg).catch(console.error);
+        }
+      } catch (e) {
+        console.error("Postgres stream poll error", e);
+      }
+
+      if (this.activeSubscriptions.get(subId)) {
+        setTimeout(poll, this.pollInterval);
+      }
+    };
+
+    setTimeout(poll, 0);
+
+    return {
+      unsubscribe: async () => {
+        this.activeSubscriptions.set(subId, false);
+      },
+    };
+  }
+
+  async createConsumerGroup(stream: string, group: string): Promise<void> {
+    // Manage in DB
+  }
+
+  async saveWorkflowState(
+    workflowId: string,
+    state: WorkflowExecution
+  ): Promise<void> {
+    // Upsert execution
+    await this.db
+      .insert(workflowExecutions)
+      .values({
+        id: state.id,
+        workflowId: state.workflowId,
+        status: state.status,
+        input: state.input,
+        output: state.output,
+        error:
+          state.error instanceof Error
+            ? state.error.message
+            : String(state.error),
+        startedAt: state.startedAt,
+        completedAt: state.completedAt,
+      })
+      .onConflictDoUpdate({
+        target: workflowExecutions.id,
+        set: {
+          status: state.status,
+          output: state.output,
+          error:
+            state.error instanceof Error
+              ? state.error.message
+              : String(state.error),
+          updatedAt: Date.now(),
+          completedAt: state.completedAt,
+        },
+      });
+  }
+
+  async loadWorkflowState(
+    executionId: string
+  ): Promise<WorkflowExecution | null> {
+    const rows = await this.db
+      .select()
+      .from(workflowExecutions)
+      .where(eq(workflowExecutions.id, executionId));
+    if (rows.length === 0) return null;
+    const row = rows[0];
+    return {
+      id: row.id,
+      workflowId: row.workflowId,
+      status: row.status as any, // Cast from DB string to union type
+      input: row.input,
+      output: row.output,
+      error: row.error,
+      startedAt: Number(row.startedAt),
+      completedAt: row.completedAt ? Number(row.completedAt) : undefined,
+    };
+  }
+
+  async getJob(queue: string, jobId: string): Promise<Job | null> {
+    const rows = await this.db
+      .select()
+      .from(jobs)
+      .where(and(eq(jobs.queue, queue), eq(jobs.id, jobId)));
+    if (rows.length === 0) return null;
+    const row = rows[0];
+    return {
+      id: row.id,
+      name: row.name,
+      data: row.data,
+      opts: row.opts,
+      state: row.state,
+      timestamp: Number(row.timestamp),
+      attemptsMade: row.attempts_made,
+    } as any;
+  }
+
+  async getQueueStats(queueName: string): Promise<QueueStats> {
+    const counts = await this.db
+      .select({
+        state: jobs.state,
+        count: sql<number>`count(*)`,
+      })
+      .from(jobs)
+      .where(eq(jobs.queue, queueName))
+      .groupBy(jobs.state);
+
+    const stats: QueueStats = {
+      waiting: 0,
+      active: 0,
+      completed: 0,
+      failed: 0,
+      delayed: 0,
+      paused: 0,
+    };
+    for (const c of counts) {
+      if (c.state in stats) {
+        (stats as any)[c.state] = Number(c.count);
+      }
+    }
+    return stats;
+  }
+
+  async getStreamInfo(streamName: string): Promise<StreamInfo> {
+    return { name: streamName, length: 0, groups: 0 };
+  }
+
+  async consume(
+    stream: string,
+    group: string,
+    consumer: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription> {
+    // Simplified consumer group implementation
+    return this.subscribe(stream, handler);
+  }
+
+  async getJobs(queue: string, status: string): Promise<Job[]> {
+    const rows = await this.db
+      .select()
+      .from(jobs)
+      .where(and(eq(jobs.queue, queue), eq(jobs.state, status)));
+    return rows.map(
+      (row: any) =>
+        ({
+          id: row.id,
+          name: row.name,
+          data: row.data,
+          opts: row.opts,
+          state: row.state,
+          timestamp: Number(row.timestamp),
+          attemptsMade: row.attempts_made,
+        }) as any
+    );
+  }
+
+  async getAllJobs(queue: string): Promise<Job[]> {
+    const rows = await this.db.select().from(jobs).where(eq(jobs.queue, queue));
+    return rows.map(
+      (row: any) =>
+        ({
+          id: row.id,
+          name: row.name,
+          data: row.data,
+          opts: row.opts,
+          state: row.state,
+          timestamp: Number(row.timestamp),
+          attemptsMade: row.attempts_made,
+        }) as any
+    );
+  }
+
+  async cleanJobs(
+    queue: string,
+    grace: number,
+    status: string
+  ): Promise<number> {
+    const now = Date.now();
+    const result = await this.db
+      .delete(jobs)
+      .where(
+        and(
+          eq(jobs.queue, queue),
+          eq(jobs.state, status),
+          sql`${jobs.finishedOn} < ${now - grace}`
+        )
+      );
+    return result.rowCount || 0;
+  }
+
+  async cleanup(): Promise<void> {
+    this.activeSubscriptions.clear();
+  }
+}

package/src/adapters/postgres/postgres.test.ts

@@ -0,0 +1,100 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { PostgresAdapter } from './index.js';
+import { JobImpl } from '../../queue/job.js';
+import { drizzle } from 'drizzle-orm/better-sqlite3';
+import Database from 'better-sqlite3';
+import { jobs, messages, workflowExecutions } from './schema.js';
+import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
+import { sql } from 'drizzle-orm';
+import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core';
+
+// We need to verify compatibility. PostgresAdapter uses Postgres specific SQL (SKIP LOCKED).
+// Testing with SQLite won't work for enqueue/dequeue if we use raw SQL SKIP LOCKED.
+// However, for unit testing logic without a real Postgres, we might need to mock db.execute 
+// OR implement a fallback in the adapter for non-Postgres (which we kind of did but with raw SQL).
+
+// Since we can't easily spin up Postgres in this environment without docker, 
+// and the adapter has hardcoded Postgres SQL for dequeue, 
+// we will Mock the DB instance.
+
+describe('PostgresAdapter', () => {
+    let adapter: PostgresAdapter;
+    let mockDb: any;
+
+    beforeEach(() => {
+        mockDb = {
+            insert: () => ({ values: () => Promise.resolve() }), // mock insert
+            update: () => ({ set: () => ({ where: () => Promise.resolve() }) }),
+            select: () => ({ from: () => ({ where: () => ({ groupBy: () => Promise.resolve([]), orderBy: () => ({ limit: () => Promise.resolve([]) }) }), limit: () => Promise.resolve([]) }) }), // mock select chain
+            transaction: async (cb: any) => cb(mockDb),
+            execute: async () => [],
+        };
+        adapter = new PostgresAdapter({ db: mockDb });
+    });
+
+    it('should enqueue a job', async () => {
+        // Just verify it calls insert
+        let inserted = false;
+        mockDb.insert = (table) => {
+            return {
+                values: (vals) => {
+                    inserted = true;
+                    expect(vals.queue).toBe('test-queue');
+                    return Promise.resolve();
+                }
+            };
+        };
+        
+        const job = new JobImpl('test-job', { data: 123 });
+        await adapter.enqueue('test-queue', job);
+        expect(inserted).toBe(true);
+    });
+
+    it('should dequeue a job (simulated)', async () => {
+        // Mock execute to return a job
+        mockDb.execute = async (query) => {
+            // Check if it's the dequeue query
+            // Drizzle sql`` returns an object with sql/params or just strings array
+            // The exact structure depends on implementation but for mocking we check what we get
+            const sqlStr = query.sql || (query.strings ? query.strings.join('') : String(query));
+            
+            if (sqlStr.includes('UPDATE flowfn_jobs') || JSON.stringify(query).includes('UPDATE flowfn_jobs')) {
+                return [{
+                    id: 'job-1',
+                    queue: 'test-queue',
+                    name: 'test-job',
+                    data: { data: 123 },
+                    opts: {},
+                    state: 'active',
+                    timestamp: Date.now()
+                }];
+            }
+            return [];
+        };
+
+        const job = await adapter.dequeue('test-queue');
+        expect(job).toBeDefined();
+        expect(job!.id).toBe('job-1');
+    });
+
+    it('should save workflow state', async () => {
+        let upserted = false;
+        mockDb.insert = () => ({
+            values: (v) => ({
+                onConflictDoUpdate: (opts) => {
+                    upserted = true;
+                    return Promise.resolve();
+                }
+            })
+        });
+
+        await adapter.saveWorkflowState('wf-1', {
+            id: 'exec-1',
+            workflowId: 'wf-1',
+            status: 'running',
+            startedAt: Date.now()
+        });
+        
+        expect(upserted).toBe(true);
+    });
+});