flowfn 0.0.1

package/src/adapters/postgres/schema.ts

@@ -0,0 +1,110 @@
+import { pgTable, varchar, text, jsonb, integer, bigint, timestamp, index, uniqueIndex } from 'drizzle-orm/pg-core';
+
+export const jobs = pgTable('flowfn_jobs', {
+  id: varchar('id', { length: 255 }).primaryKey(),
+  queue: varchar('queue', { length: 255 }).notNull(),
+  name: varchar('name', { length: 255 }).notNull(),
+  data: jsonb('data').notNull(),
+  opts: jsonb('opts'),
+  
+  state: varchar('state', { length: 50 }).notNull(), // waiting, active, completed, failed, delayed, paused
+  priority: integer('priority').default(0),
+  progress: integer('progress').default(0),
+  returnValue: jsonb('return_value'),
+  
+  timestamp: bigint('timestamp', { mode: 'number' }).notNull(),
+  processedOn: bigint('processed_on', { mode: 'number' }),
+  finishedOn: bigint('finished_on', { mode: 'number' }),
+  delay: bigint('delay', { mode: 'number' }).default(0),
+  
+  attemptsMade: integer('attempts_made').default(0),
+  failedReason: text('failed_reason'),
+  stacktrace: jsonb('stacktrace'),
+  
+  createdAt: timestamp('created_at').defaultNow(),
+  updatedAt: timestamp('updated_at').defaultNow(),
+}, (table) => ({
+  queueStateIdx: index('idx_queue_state').on(table.queue, table.state),
+  queuePriorityIdx: index('idx_queue_priority').on(table.queue, table.priority),
+  timestampIdx: index('idx_timestamp').on(table.timestamp),
+}));
+
+export const messages = pgTable('flowfn_messages', {
+  id: varchar('id', { length: 255 }).primaryKey(),
+  stream: varchar('stream', { length: 255 }).notNull(),
+  data: jsonb('data').notNull(),
+  headers: jsonb('headers'),
+  
+  partition: integer('partition'),
+  offset: bigint('offset', { mode: 'number' }),
+  key: varchar('key', { length: 255 }),
+  
+  timestamp: bigint('timestamp', { mode: 'number' }).notNull(),
+  createdAt: timestamp('created_at').defaultNow(),
+}, (table) => ({
+  streamTimestampIdx: index('idx_stream_timestamp').on(table.stream, table.timestamp),
+}));
+
+export const consumerGroups = pgTable('flowfn_consumer_groups', {
+  id: varchar('id', { length: 255 }).primaryKey(),
+  stream: varchar('stream', { length: 255 }).notNull(),
+  groupId: varchar('group_id', { length: 255 }).notNull(),
+  consumerId: varchar('consumer_id', { length: 255 }).notNull(),
+  
+  lastMessageId: varchar('last_message_id', { length: 255 }),
+  lastOffset: bigint('last_offset', { mode: 'number' }),
+  lag: integer('lag'),
+  
+  createdAt: timestamp('created_at').defaultNow(),
+  updatedAt: timestamp('updated_at').defaultNow(),
+}, (table) => ({
+  streamGroupConsumerIdx: uniqueIndex('idx_stream_group_consumer').on(table.stream, table.groupId, table.consumerId),
+}));
+
+export const workflows = pgTable('flowfn_workflows', {
+  id: varchar('id', { length: 255 }).primaryKey(),
+  name: varchar('name', { length: 255 }).notNull(),
+  definition: jsonb('definition').notNull(),
+  version: integer('version').default(1),
+  
+  status: varchar('status', { length: 50 }).notNull(),
+  metadata: jsonb('metadata'),
+  
+  createdAt: timestamp('created_at').defaultNow(),
+  updatedAt: timestamp('updated_at').defaultNow(),
+});
+
+export const workflowExecutions = pgTable('flowfn_workflow_executions', {
+  id: varchar('id', { length: 255 }).primaryKey(),
+  workflowId: varchar('workflow_id', { length: 255 }).notNull(),
+  workflowName: varchar('workflow_name', { length: 255 }),
+  
+  status: varchar('status', { length: 50 }).notNull(),
+  input: jsonb('input'),
+  state: jsonb('state'),
+  output: jsonb('output'),
+  error: text('error'),
+  
+  currentStep: varchar('current_step', { length: 255 }),
+  completedSteps: jsonb('completed_steps'),
+  
+  startedAt: bigint('started_at', { mode: 'number' }).notNull(),
+  updatedAt: bigint('updated_at', { mode: 'number' }),
+  completedAt: bigint('completed_at', { mode: 'number' }),
+  durationMs: integer('duration_ms'),
+}, (table) => ({
+  workflowIdx: index('idx_workflow').on(table.workflowId),
+  statusIdx: index('idx_status').on(table.status),
+}));
+
+export const workflowEvents = pgTable('flowfn_workflow_events', {
+  id: varchar('id', { length: 255 }).primaryKey(),
+  executionId: varchar('execution_id', { length: 255 }).notNull(),
+  
+  type: varchar('type', { length: 50 }).notNull(),
+  step: varchar('step', { length: 255 }),
+  timestamp: bigint('timestamp', { mode: 'number' }).notNull(),
+  data: jsonb('data'),
+}, (table) => ({
+  executionIdx: index('idx_execution').on(table.executionId),
+}));

package/src/adapters/redis.test.ts

@@ -0,0 +1,124 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { RedisAdapter } from './redis.js';
+import { JobImpl } from '../queue/job.js';
+
+// Mock ioredis to return RedisMock
+vi.mock('ioredis', async (importOriginal) => {
+    const RedisMock = (await import('ioredis-mock')).default;
+    return {
+        default: RedisMock,
+    };
+});
+
+describe('RedisAdapter', () => {
+    let adapter: RedisAdapter;
+    let redis: any;
+
+    beforeEach(async () => {
+        const RedisMock = (await import('ioredis-mock')).default;
+        redis = new RedisMock();
+        
+        // Polyfill missing stream commands for ioredis-mock
+        redis.xgroup = vi.fn().mockResolvedValue('OK');
+        redis.xack = vi.fn().mockResolvedValue(1);
+        
+        // Mock xreadgroup to behave like xread for testing purposes
+        // In reality, it should filter by group/consumer, but for this unit test
+        // we just want to verify data flow.
+        redis.xreadgroup = vi.fn().mockImplementation(async (cmd, group, consumer, block, ms, count, n, streams, key, id) => {
+            // Mapping arguments is tricky because ioredis passes them as varargs
+            // The call in adapter is: xreadgroup('GROUP', group, consumer, 'BLOCK', 1000, 'COUNT', 10, 'STREAMS', streamKey, '>')
+            // We'll just call xread internally to get data
+            // xread('BLOCK', 1000, 'STREAMS', streamKey, '0-0') to get all
+            
+            // Extract stream key - it's usually near the end
+            // The adapter calls it as: .xreadgroup('GROUP', group, consumer, 'BLOCK', 1000, 'COUNT', 10, 'STREAMS', streamKey, '>')
+            
+            // Simulating a response
+            const data = await redis.xread('STREAMS', 'flowfn:stream:group-stream', '0-0');
+            return data;
+        });
+
+        adapter = new RedisAdapter({ connection: redis });
+    });
+
+    afterEach(async () => {
+        await adapter.cleanup();
+    });
+
+    it('should enqueue and dequeue a job', async () => {
+        const job = new JobImpl('test-job', { data: 123 });
+        await adapter.enqueue('test-queue', job);
+        
+        const dequeued = await adapter.dequeue('test-queue');
+        expect(dequeued).toBeDefined();
+        expect(dequeued!.id).toBe(job.id);
+        expect(dequeued!.data).toEqual({ data: 123 });
+    });
+
+    it('should handle delayed jobs', async () => {
+        const job = new JobImpl('delayed-job', { data: 'late' }, { delay: 1000 });
+        await adapter.enqueue('test-queue', job);
+        
+        // Should be empty initially
+        const immediate = await adapter.dequeue('test-queue');
+        expect(immediate).toBeNull();
+
+        const delayedKey = 'flowfn:delayed:test-queue';
+        const delayedJobs = await redis.zcard(delayedKey);
+        expect(delayedJobs).toBe(1);
+    });
+
+    it('should publish and subscribe to a stream (XADD/XREAD)', async () => {
+        const received: any[] = [];
+        await adapter.subscribe('test-stream', async (msg) => {
+            received.push(msg);
+        });
+
+        const message = { 
+            id: '1', 
+            stream: 'test-stream', 
+            data: { hello: 'world' }, 
+            headers: { foo: 'bar' },
+            timestamp: Date.now(), 
+            ack: async () => {}, 
+            nack: async () => {} 
+        };
+        await adapter.publish('test-stream', message);
+
+        // Wait for polling loop
+        await new Promise(r => setTimeout(r, 200));
+        expect(received.length).toBeGreaterThan(0);
+        expect(received[0].data).toEqual({ hello: 'world' });
+        expect(received[0].headers).toEqual({ foo: 'bar' });
+    });
+
+    it('should consume via consumer group', async () => {
+        const received: any[] = [];
+        
+        await adapter.consume('group-stream', 'mygroup', 'consumer1', async (msg) => {
+            received.push(msg);
+            await msg.ack();
+        });
+
+        const message = { 
+            id: '1', 
+            stream: 'group-stream', 
+            data: { group: 'test' }, 
+            timestamp: Date.now(), 
+            ack: async () => {}, 
+            nack: async () => {} 
+        };
+        await adapter.publish('group-stream', message);
+
+        // Wait for polling loop
+        await new Promise(r => setTimeout(r, 200));
+        
+        expect(received.length).toBeGreaterThan(0);
+        expect(received[0].data).toEqual({ group: 'test' });
+        
+        // Verify xgroup/xack called
+        expect(redis.xgroup).toHaveBeenCalled();
+        expect(redis.xack).toHaveBeenCalled();
+    });
+});

package/src/adapters/redis.ts

@@ -0,0 +1,331 @@
+import { FlowAdapter } from "./base.js";
+import { Job, QueueStats } from "../queue/types.js";
+import {
+  Message,
+  MessageHandler,
+  Subscription,
+  StreamInfo,
+} from "../stream/types.js";
+import { WorkflowExecution } from "../workflow/types.js";
+import Redis, { RedisOptions } from "ioredis";
+
+export interface RedisAdapterOptions {
+  connection?: Redis | RedisOptions;
+  prefix?: string;
+}
+
+export class RedisAdapter implements FlowAdapter {
+  private redis: Redis;
+  private prefix: string;
+  private subscriptions: Map<string, Redis[]> = new Map();
+
+  constructor(options: RedisAdapterOptions = {}) {
+    if (options.connection instanceof Redis) {
+      this.redis = options.connection;
+    } else {
+      this.redis = new Redis((options.connection as RedisOptions) || {});
+    }
+    this.prefix = options.prefix || "flowfn:";
+  }
+
+  private key(type: string, name: string): string {
+    return `${this.prefix}${type}:${name}`;
+  }
+
+  async enqueue(queueName: string, job: Job): Promise<string> {
+    const queueKey = this.key("queue", queueName);
+    const jobKey = this.key("job", job.id);
+
+    // Store job data
+    await this.redis.set(jobKey, JSON.stringify(job));
+
+    if (job.opts.delay && job.opts.delay > 0) {
+      const delayedKey = this.key("delayed", queueName);
+      await this.redis.zadd(delayedKey, Date.now() + job.opts.delay, job.id);
+    } else {
+      await this.redis.lpush(queueKey, job.id);
+    }
+
+    return job.id;
+  }
+
+  async dequeue(queueName: string): Promise<Job | null> {
+    const queueKey = this.key("queue", queueName);
+    const delayedKey = this.key("delayed", queueName);
+
+    // Check delayed jobs first
+    const now = Date.now();
+    const delayed = await this.redis.zrangebyscore(
+      delayedKey,
+      0,
+      now,
+      "LIMIT",
+      0,
+      1
+    );
+    if (delayed.length > 0) {
+      const jobId = delayed[0];
+      await this.redis.zrem(delayedKey, jobId);
+      await this.redis.lpush(queueKey, jobId);
+    }
+
+    const jobId = await this.redis.rpop(queueKey);
+    if (!jobId) return null;
+
+    const jobData = await this.redis.get(this.key("job", jobId));
+    if (!jobData) return null;
+
+    return JSON.parse(jobData);
+  }
+
+  async ack(queue: string, jobId: string): Promise<void> {
+    await this.redis.del(this.key("job", jobId));
+  }
+
+  async nack(
+    queueName: string,
+    jobId: string,
+    requeue: boolean = true
+  ): Promise<void> {
+    if (requeue) {
+      await this.redis.lpush(this.key("queue", queueName), jobId);
+    }
+  }
+
+  async publish(streamName: string, message: Message): Promise<string> {
+    const streamKey = this.key("stream", streamName);
+    const id = await this.redis.xadd(
+      streamKey,
+      "*",
+      "data",
+      JSON.stringify(message.data),
+      "headers",
+      JSON.stringify(message.headers || {})
+    );
+    return id || "";
+  }
+
+  async subscribe(
+    streamName: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription> {
+    const streamKey = this.key("stream", streamName);
+    let active = true;
+
+    // Simple polling for new messages from '$' (end)
+    // Real impl might use blocking XREAD but that requires a dedicated connection or careful management
+    // We'll use a polling loop with XREAD for simplicity on shared connection
+    let lastId = "$";
+
+    const poll = async () => {
+      if (!active) return;
+      try {
+        const results = (await this.redis.xread(
+          "STREAMS",
+          streamKey,
+          lastId,
+          "BLOCK",
+          1000
+        )) as any;
+        if (results) {
+          for (const [stream, messages] of results) {
+            for (const [id, fields] of messages) {
+              lastId = id;
+              // fields is array [key1, val1, key2, val2...]
+              const dataIdx = fields.indexOf("data");
+              const headersIdx = fields.indexOf("headers");
+
+              const data = dataIdx > -1 ? JSON.parse(fields[dataIdx + 1]) : {};
+              const headers =
+                headersIdx > -1 ? JSON.parse(fields[headersIdx + 1]) : {};
+
+              const msg: Message = {
+                id,
+                stream: streamName,
+                data,
+                headers,
+                timestamp: parseInt(id.split("-")[0]),
+                ack: async () => {},
+                nack: async () => {},
+              };
+              handler(msg).catch(console.error);
+            }
+          }
+        }
+      } catch (err) {
+        console.error("Redis stream poll error", err);
+        await new Promise((r) => setTimeout(r, 1000));
+      }
+      if (active) setTimeout(poll, 0);
+    };
+
+    // Start polling in next tick
+    setTimeout(poll, 0);
+
+    return {
+      unsubscribe: async () => {
+        active = false;
+      },
+    };
+  }
+
+  async consume(
+    streamName: string,
+    group: string,
+    consumer: string,
+    handler: MessageHandler<any>
+  ): Promise<Subscription> {
+    const streamKey = this.key("stream", streamName);
+
+    // Ensure group exists
+    try {
+      await this.redis.xgroup("CREATE", streamKey, group, "$", "MKSTREAM");
+    } catch (e: any) {
+      if (!e.message.includes("BUSYGROUP")) throw e;
+    }
+
+    let active = true;
+    const poll = async () => {
+      if (!active) return;
+      try {
+        // Read as consumer group
+        const results = (await this.redis.xreadgroup(
+          "GROUP",
+          group,
+          consumer,
+          "COUNT",
+          10,
+          "BLOCK",
+          1000,
+          "STREAMS",
+          streamKey,
+          ">"
+        )) as any;
+
+        if (results) {
+          for (const [stream, messages] of results) {
+            for (const [id, fields] of messages) {
+              const dataIdx = fields.indexOf("data");
+              const headersIdx = fields.indexOf("headers");
+              const data = dataIdx > -1 ? JSON.parse(fields[dataIdx + 1]) : {};
+              const headers =
+                headersIdx > -1 ? JSON.parse(fields[headersIdx + 1]) : {};
+
+              const msg: Message = {
+                id,
+                stream: streamName,
+                data,
+                headers,
+                timestamp: parseInt(id.split("-")[0]),
+                ack: async () => {
+                  await this.redis.xack(streamKey, group, id);
+                },
+                nack: async (requeue = true) => {
+                  // Redis streams don't support NACK/requeue natively like AMQP.
+                  // The message stays in PEL (Pending Entries List).
+                  // We could claim it or just leave it for another consumer to claim.
+                },
+              };
+              handler(msg).catch(console.error);
+            }
+          }
+        }
+      } catch (err) {
+        console.error("Redis consumer poll error", err);
+        await new Promise((r) => setTimeout(r, 1000));
+      }
+      if (active) setTimeout(poll, 0);
+    };
+
+    setTimeout(poll, 0);
+
+    return {
+      unsubscribe: async () => {
+        active = false;
+      },
+    };
+  }
+
+  async createConsumerGroup(stream: string, group: string): Promise<void> {
+    const streamKey = this.key("stream", stream);
+    try {
+      await this.redis.xgroup("CREATE", streamKey, group, "$", "MKSTREAM");
+    } catch (e: any) {
+      if (!e.message.includes("BUSYGROUP")) throw e;
+    }
+  }
+
+  async saveWorkflowState(
+    workflowId: string,
+    state: WorkflowExecution
+  ): Promise<void> {
+    await this.redis.set(
+      this.key("workflow", workflowId),
+      JSON.stringify(state)
+    );
+  }
+
+  async loadWorkflowState(
+    workflowId: string
+  ): Promise<WorkflowExecution | null> {
+    const data = await this.redis.get(this.key("workflow", workflowId));
+    return data ? JSON.parse(data) : null;
+  }
+
+  async getJob(queue: string, jobId: string): Promise<Job | null> {
+    const jobData = await this.redis.get(this.key("job", jobId));
+    return jobData ? JSON.parse(jobData) : null;
+  }
+
+  async getJobs(queue: string, status: string): Promise<Job[]> {
+    // Simplified implementation - would need custom tracking for status in production
+    return [];
+  }
+
+  async getAllJobs(queue: string): Promise<Job[]> {
+    // Simplified implementation - would need custom tracking in production
+    return [];
+  }
+
+  async cleanJobs(
+    queue: string,
+    grace: number,
+    status: string
+  ): Promise<number> {
+    // Simplified implementation - would need custom tracking in production
+    return 0;
+  }
+
+  async getQueueStats(queueName: string): Promise<QueueStats> {
+    const length = await this.redis.llen(this.key("queue", queueName));
+    const delayedLength = await this.redis.zcard(
+      this.key("delayed", queueName)
+    );
+    return {
+      waiting: length,
+      active: 0,
+      completed: 0,
+      failed: 0,
+      delayed: delayedLength,
+      paused: 0,
+    };
+  }
+
+  async getStreamInfo(streamName: string): Promise<StreamInfo> {
+    return {
+      name: streamName,
+      length: 0,
+      groups: 0,
+    };
+  }
+
+  async cleanup(): Promise<void> {
+    for (const subs of this.subscriptions.values()) {
+      for (const sub of subs) {
+        await sub.quit();
+      }
+    }
+    this.subscriptions.clear();
+    await this.redis.quit();
+  }
+}

package/src/core/flow-fn.test.ts

@@ -0,0 +1,70 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { createFlow } from './flow-fn.js';
+
+describe('flowfn', () => {
+    let flow;
+
+    beforeEach(() => {
+        flow = createFlow({ adapter: 'memory' });
+    });
+
+    afterEach(async () => {
+        await flow.close();
+    });
+
+    it('should process jobs in a queue', async () => {
+        const queue = flow.queue('test-queue');
+        
+        // Add job
+        await queue.add('job1', { foo: 'bar' });
+        
+        const stats = await queue.getJobCounts();
+        expect(stats.waiting).toBe(1);
+
+        // Process
+        await new Promise<void>(resolve => {
+            queue.process(async (job) => {
+                expect(job.data.foo).toBe('bar');
+                resolve();
+                return { processed: true };
+            });
+        });
+    });
+
+    it('should stream messages', async () => {
+        const stream = flow.stream('test-stream');
+        const received: any[] = [];
+
+        await stream.subscribe(async (msg) => {
+            received.push(msg.data);
+        });
+
+        await stream.publish({ event: 'test' });
+        
+        // Wait for event loop
+        await new Promise(r => setTimeout(r, 50));
+        
+        expect(received).toHaveLength(1);
+        expect(received[0]).toEqual({ event: 'test' });
+    });
+
+    it('should execute a simple workflow', async () => {
+        const workflow = flow.workflow('test-workflow')
+            .step('step1', async (ctx) => {
+                await ctx.sleep(100); // Wait so we can catch it running
+                ctx.set('step1', 'done');
+                return 'result1';
+            })
+            .build();
+
+        const execution = await workflow.execute({ input: 'test' });
+        
+        expect(execution.status).toBe('running');
+        
+        // Wait for execution
+        await new Promise(r => setTimeout(r, 150));
+        
+        const status = await workflow.getExecution(execution.id);
+        expect(status.status).toBe('completed');
+    });
+});