flowfn 0.0.1

package/src/queue/worker.ts

@@ -0,0 +1,66 @@
+import { FlowFn } from '../core/flow-fn.js';
+import { Queue, Job } from './types.js';
+import EventEmitter from 'eventemitter3';
+
+export interface WorkerOptions {
+  flow: FlowFn;
+  queues: string[];
+  concurrency?: number | Record<string, number>;
+  settings?: {
+      stalledInterval?: number;
+      maxStalledCount?: number;
+  };
+}
+
+export class Worker extends EventEmitter {
+  private flow: FlowFn;
+  private queueNames: string[];
+  private concurrency: number | Record<string, number>;
+  private queues: Queue[] = [];
+
+  constructor(options: WorkerOptions) {
+    super();
+    this.flow = options.flow;
+    this.queueNames = options.queues;
+    this.concurrency = options.concurrency || 1;
+  }
+
+  async run(): Promise<void> {
+    for (const name of this.queueNames) {
+        const queue = this.flow.queue(name);
+        this.queues.push(queue);
+        
+        const qConcurrency = typeof this.concurrency === 'number' 
+            ? this.concurrency 
+            : (this.concurrency[name] || 1);
+
+        queue.on('active', (job) => this.emit('active', job));
+        queue.on('completed', (job, result) => this.emit('completed', job, result));
+        queue.on('failed', (job, err) => this.emit('failed', job, err));
+        
+        // This is a bit simplified. In a real system, the worker would 
+        // have its own processing loop rather than calling queue.process.
+        // But for this abstraction it works.
+        // We need a way to pass the handler to the worker.
+    }
+    this.emit('ready');
+  }
+
+  // Helper to register handlers
+  register(queueName: string, handler: (job: Job) => Promise<any>) {
+      const queue = this.flow.queue(queueName);
+      const qConcurrency = typeof this.concurrency === 'number' 
+            ? this.concurrency 
+            : (this.concurrency[queueName] || 1);
+      queue.process(qConcurrency, handler);
+  }
+
+  async close(): Promise<void> {
+    this.emit('closing');
+    await Promise.all(this.queues.map(q => q.close()));
+  }
+}
+
+export function createWorker(options: WorkerOptions): Worker {
+    return new Worker(options);
+}

package/src/storage/event-log.ts

@@ -0,0 +1,205 @@
+/**
+ * Event log for event sourcing pattern
+ */
+
+export interface Event {
+  id: string;
+  type: string;
+  aggregateId: string;
+  aggregateType: string;
+  data: any;
+  metadata?: Record<string, any>;
+  timestamp: number;
+  version: number;
+}
+
+export interface EventFilter {
+  aggregateId?: string;
+  aggregateType?: string;
+  types?: string[];
+  fromVersion?: number;
+  toVersion?: number;
+  fromTimestamp?: number;
+  toTimestamp?: number;
+  limit?: number;
+  offset?: number;
+}
+
+export interface EventLog {
+  /**
+   * Append event to the log
+   */
+  append(event: Omit<Event, "id" | "timestamp" | "version">): Promise<Event>;
+
+  /**
+   * Get events by filter
+   */
+  getEvents(filter: EventFilter): Promise<Event[]>;
+
+  /**
+   * Get events for an aggregate
+   */
+  getAggregateEvents(
+    aggregateId: string,
+    fromVersion?: number
+  ): Promise<Event[]>;
+
+  /**
+   * Get latest version for an aggregate
+   */
+  getLatestVersion(aggregateId: string): Promise<number>;
+
+  /**
+   * Subscribe to event stream
+   */
+  subscribe(
+    handler: (event: Event) => void | Promise<void>,
+    filter?: EventFilter
+  ): () => void;
+}
+
+/**
+ * In-memory event log implementation
+ */
+export class MemoryEventLog implements EventLog {
+  private events: Event[] = [];
+  private aggregateVersions: Map<string, number> = new Map();
+  private subscribers: Array<{
+    handler: (event: Event) => void | Promise<void>;
+    filter?: EventFilter;
+  }> = [];
+
+  async append(
+    event: Omit<Event, "id" | "timestamp" | "version">
+  ): Promise<Event> {
+    const version = (this.aggregateVersions.get(event.aggregateId) || 0) + 1;
+    this.aggregateVersions.set(event.aggregateId, version);
+
+    const fullEvent: Event = {
+      ...event,
+      id: `evt_${Date.now()}_${Math.random().toString(36).substring(7)}`,
+      timestamp: Date.now(),
+      version,
+    };
+
+    this.events.push(fullEvent);
+
+    // Notify subscribers
+    for (const subscriber of this.subscribers) {
+      if (this.matchesFilter(fullEvent, subscriber.filter)) {
+        Promise.resolve(subscriber.handler(fullEvent)).catch((err) =>
+          console.error("Event subscriber error:", err)
+        );
+      }
+    }
+
+    return fullEvent;
+  }
+
+  async getEvents(filter: EventFilter): Promise<Event[]> {
+    let results = [...this.events];
+
+    // Apply filters
+    if (filter.aggregateId) {
+      results = results.filter((e) => e.aggregateId === filter.aggregateId);
+    }
+
+    if (filter.aggregateType) {
+      results = results.filter((e) => e.aggregateType === filter.aggregateType);
+    }
+
+    if (filter.types && filter.types.length > 0) {
+      results = results.filter((e) => filter.types!.includes(e.type));
+    }
+
+    if (filter.fromVersion !== undefined) {
+      results = results.filter((e) => e.version >= filter.fromVersion!);
+    }
+
+    if (filter.toVersion !== undefined) {
+      results = results.filter((e) => e.version <= filter.toVersion!);
+    }
+
+    if (filter.fromTimestamp !== undefined) {
+      results = results.filter((e) => e.timestamp >= filter.fromTimestamp!);
+    }
+
+    if (filter.toTimestamp !== undefined) {
+      results = results.filter((e) => e.timestamp <= filter.toTimestamp!);
+    }
+
+    // Sort by version
+    results.sort((a, b) => a.version - b.version);
+
+    // Pagination
+    const offset = filter.offset || 0;
+    const limit = filter.limit || results.length;
+
+    return results.slice(offset, offset + limit);
+  }
+
+  async getAggregateEvents(
+    aggregateId: string,
+    fromVersion?: number
+  ): Promise<Event[]> {
+    return this.getEvents({
+      aggregateId,
+      fromVersion,
+    });
+  }
+
+  async getLatestVersion(aggregateId: string): Promise<number> {
+    return this.aggregateVersions.get(aggregateId) || 0;
+  }
+
+  subscribe(
+    handler: (event: Event) => void | Promise<void>,
+    filter?: EventFilter
+  ): () => void {
+    const subscriber = { handler, filter };
+    this.subscribers.push(subscriber);
+
+    // Return unsubscribe function
+    return () => {
+      const index = this.subscribers.indexOf(subscriber);
+      if (index > -1) {
+        this.subscribers.splice(index, 1);
+      }
+    };
+  }
+
+  private matchesFilter(event: Event, filter?: EventFilter): boolean {
+    if (!filter) return true;
+
+    if (filter.aggregateId && event.aggregateId !== filter.aggregateId)
+      return false;
+    if (filter.aggregateType && event.aggregateType !== filter.aggregateType)
+      return false;
+    if (filter.types && !filter.types.includes(event.type)) return false;
+    if (filter.fromVersion !== undefined && event.version < filter.fromVersion)
+      return false;
+    if (filter.toVersion !== undefined && event.version > filter.toVersion)
+      return false;
+    if (
+      filter.fromTimestamp !== undefined &&
+      event.timestamp < filter.fromTimestamp
+    )
+      return false;
+    if (
+      filter.toTimestamp !== undefined &&
+      event.timestamp > filter.toTimestamp
+    )
+      return false;
+
+    return true;
+  }
+
+  /**
+   * Clear all events
+   */
+  clear(): void {
+    this.events = [];
+    this.aggregateVersions.clear();
+    this.subscribers = [];
+  }
+}

package/src/storage/job-storage.ts

@@ -0,0 +1,206 @@
+/**
+ * Storage abstraction for FlowFn job persistence
+ */
+
+import { Job, JobStatus } from "../queue/types.js";
+
+export interface JobStorage {
+  /**
+   * Save a job
+   */
+  save(queue: string, job: Job): Promise<void>;
+
+  /**
+   * Get a job by ID
+   */
+  get(queue: string, jobId: string): Promise<Job | null>;
+
+  /**
+   * Get jobs by status
+   */
+  getByStatus(queue: string, status: JobStatus): Promise<Job[]>;
+
+  /**
+   * Get all jobs for a queue
+   */
+  getAll(queue: string): Promise<Job[]>;
+
+  /**
+   * Update job status
+   */
+  updateStatus(queue: string, jobId: string, status: JobStatus): Promise<void>;
+
+  /**
+   * Update job data
+   */
+  update(queue: string, jobId: string, updates: Partial<Job>): Promise<void>;
+
+  /**
+   * Delete a job
+   */
+  delete(queue: string, jobId: string): Promise<void>;
+
+  /**
+   * Delete jobs older than grace period with given status
+   */
+  clean(queue: string, grace: number, status: JobStatus): Promise<number>;
+
+  /**
+   * Get job count by status
+   */
+  count(queue: string, status?: JobStatus): Promise<number>;
+
+  /**
+   * Check if job exists with deduplication key
+   */
+  existsByDeduplicationKey(queue: string, key: string): Promise<boolean>;
+}
+
+/**
+ * In-memory implementation of JobStorage
+ */
+export class MemoryJobStorage implements JobStorage {
+  private jobs: Map<string, Job> = new Map();
+  private deduplicationKeys: Map<string, string> = new Map(); // key -> jobId
+
+  async save(queue: string, job: Job): Promise<void> {
+    const key = `${queue}:${job.id}`;
+    this.jobs.set(key, { ...job });
+
+    // Store deduplication key if present
+    if (job.opts.deduplicationKey) {
+      const dedupKey = `${queue}:${job.opts.deduplicationKey}`;
+      this.deduplicationKeys.set(dedupKey, job.id);
+    }
+  }
+
+  async get(queue: string, jobId: string): Promise<Job | null> {
+    const key = `${queue}:${jobId}`;
+    return this.jobs.get(key) || null;
+  }
+
+  async getByStatus(queue: string, status: JobStatus): Promise<Job[]> {
+    const result: Job[] = [];
+    const prefix = `${queue}:`;
+
+    for (const [key, job] of this.jobs.entries()) {
+      if (key.startsWith(prefix) && job.state === status) {
+        result.push(job);
+      }
+    }
+
+    return result;
+  }
+
+  async getAll(queue: string): Promise<Job[]> {
+    const result: Job[] = [];
+    const prefix = `${queue}:`;
+
+    for (const [key, job] of this.jobs.entries()) {
+      if (key.startsWith(prefix)) {
+        result.push(job);
+      }
+    }
+
+    return result;
+  }
+
+  async updateStatus(
+    queue: string,
+    jobId: string,
+    status: JobStatus
+  ): Promise<void> {
+    const key = `${queue}:${jobId}`;
+    const job = this.jobs.get(key);
+
+    if (job) {
+      job.state = status;
+      if (status === "completed" || status === "failed") {
+        job.finishedOn = Date.now();
+      }
+    }
+  }
+
+  async update(
+    queue: string,
+    jobId: string,
+    updates: Partial<Job>
+  ): Promise<void> {
+    const key = `${queue}:${jobId}`;
+    const job = this.jobs.get(key);
+
+    if (job) {
+      Object.assign(job, updates);
+    }
+  }
+
+  async delete(queue: string, jobId: string): Promise<void> {
+    const key = `${queue}:${jobId}`;
+    const job = this.jobs.get(key);
+
+    if (job?.opts.deduplicationKey) {
+      const dedupKey = `${queue}:${job.opts.deduplicationKey}`;
+      this.deduplicationKeys.delete(dedupKey);
+    }
+
+    this.jobs.delete(key);
+  }
+
+  async clean(
+    queue: string,
+    grace: number,
+    status: JobStatus
+  ): Promise<number> {
+    const now = Date.now();
+    const prefix = `${queue}:`;
+    const toDelete: string[] = [];
+
+    for (const [key, job] of this.jobs.entries()) {
+      if (key.startsWith(prefix) && job.state === status) {
+        const timestamp = job.finishedOn || job.timestamp;
+        if (now - timestamp > grace) {
+          toDelete.push(key);
+        }
+      }
+    }
+
+    for (const key of toDelete) {
+      const job = this.jobs.get(key);
+      if (job?.opts.deduplicationKey) {
+        const dedupKey = `${queue}:${job.opts.deduplicationKey}`;
+        this.deduplicationKeys.delete(dedupKey);
+      }
+      this.jobs.delete(key);
+    }
+
+    return toDelete.length;
+  }
+
+  async count(queue: string, status?: JobStatus): Promise<number> {
+    const prefix = `${queue}:`;
+    let count = 0;
+
+    for (const [key, job] of this.jobs.entries()) {
+      if (key.startsWith(prefix)) {
+        if (!status || job.state === status) {
+          count++;
+        }
+      }
+    }
+
+    return count;
+  }
+
+  async existsByDeduplicationKey(queue: string, key: string): Promise<boolean> {
+    const dedupKey = `${queue}:${key}`;
+    return this.deduplicationKeys.has(dedupKey);
+  }
+
+  /**
+   * Clear all jobs
+   */
+  clear(): void {
+    this.jobs.clear();
+    this.deduplicationKeys.clear();
+  }
+}

package/src/storage/workflow-storage.ts

@@ -0,0 +1,182 @@
+/**
+ * Workflow state storage abstraction
+ */
+
+import { WorkflowExecution, ExecutionStatus } from "../workflow/types.js";
+
+export interface WorkflowStorage {
+  /**
+   * Save workflow execution state
+   */
+  save(workflowId: string, execution: WorkflowExecution): Promise<void>;
+
+  /**
+   * Get workflow execution by ID
+   */
+  get(executionId: string): Promise<WorkflowExecution | null>;
+
+  /**
+   * List executions for a workflow
+   */
+  list(workflowId: string, options?: ListOptions): Promise<WorkflowExecution[]>;
+
+  /**
+   * Update execution status
+   */
+  updateStatus(executionId: string, status: string): Promise<void>;
+
+  /**
+   * Update execution state
+   */
+  updateState(executionId: string, state: Record<string, any>): Promise<void>;
+
+  /**
+   * Delete an execution
+   */
+  delete(executionId: string): Promise<void>;
+
+  /**
+   * Clean old executions
+   */
+  clean(workflowId: string, grace: number): Promise<number>;
+}
+
+export interface ListOptions {
+  status?: string;
+  limit?: number;
+  offset?: number;
+  sortBy?: "createdAt" | "updatedAt";
+  sortOrder?: "asc" | "desc";
+}
+
+/**
+ * In-memory workflow storage implementation
+ */
+export class MemoryWorkflowStorage implements WorkflowStorage {
+  private executions: Map<string, WorkflowExecution> = new Map();
+  private workflowIndex: Map<string, Set<string>> = new Map(); // workflowId -> executionIds
+
+  async save(workflowId: string, execution: WorkflowExecution): Promise<void> {
+    this.executions.set(execution.id, { ...execution });
+
+    // Index by workflow ID
+    if (!this.workflowIndex.has(workflowId)) {
+      this.workflowIndex.set(workflowId, new Set());
+    }
+    this.workflowIndex.get(workflowId)!.add(execution.id);
+  }
+
+  async get(executionId: string): Promise<WorkflowExecution | null> {
+    return this.executions.get(executionId) || null;
+  }
+
+  async list(
+    workflowId: string,
+    options: ListOptions = {}
+  ): Promise<WorkflowExecution[]> {
+    const executionIds = this.workflowIndex.get(workflowId) || new Set();
+    let executions: WorkflowExecution[] = [];
+
+    for (const id of executionIds) {
+      const execution = this.executions.get(id);
+      if (execution) {
+        // Filter by status if specified
+        if (!options.status || execution.status === options.status) {
+          executions.push(execution);
+        }
+      }
+    }
+
+    // Sort
+    const sortBy = options.sortBy || "createdAt";
+    const sortOrder = options.sortOrder || "desc";
+
+    executions.sort((a, b) => {
+      const aVal = a[sortBy] || 0;
+      const bVal = b[sortBy] || 0;
+      return sortOrder === "asc" ? aVal - bVal : bVal - aVal;
+    });
+
+    // Pagination
+    const offset = options.offset || 0;
+    const limit = options.limit || executions.length;
+
+    return executions.slice(offset, offset + limit);
+  }
+
+  async updateStatus(
+    executionId: string,
+    status: ExecutionStatus
+  ): Promise<void> {
+    const execution = this.executions.get(executionId);
+    if (execution) {
+      execution.status = status as ExecutionStatus;
+      execution.updatedAt = Date.now();
+
+      if (
+        status === "completed" ||
+        status === "failed" ||
+        status === "cancelled"
+      ) {
+        execution.completedAt = Date.now();
+      }
+    }
+  }
+
+  async updateState(
+    executionId: string,
+    state: Record<string, any>
+  ): Promise<void> {
+    const execution = this.executions.get(executionId);
+    if (execution) {
+      execution.state = { ...execution.state, ...state };
+      execution.updatedAt = Date.now();
+    }
+  }
+
+  async delete(executionId: string): Promise<void> {
+    const execution = this.executions.get(executionId);
+    if (execution) {
+      // Remove from workflow index
+      const workflowId = execution.workflowId;
+      const executionIds = this.workflowIndex.get(workflowId);
+      if (executionIds) {
+        executionIds.delete(executionId);
+        if (executionIds.size === 0) {
+          this.workflowIndex.delete(workflowId);
+        }
+      }
+
+      this.executions.delete(executionId);
+    }
+  }
+
+  async clean(workflowId: string, grace: number): Promise<number> {
+    const now = Date.now();
+    const executionIds = this.workflowIndex.get(workflowId) || new Set();
+    const toDelete: string[] = [];
+
+    for (const id of executionIds) {
+      const execution = this.executions.get(id);
+      if (execution && execution.completedAt) {
+        if (now - execution.completedAt > grace) {
+          toDelete.push(id);
+        }
+      }
+    }
+
+    for (const id of toDelete) {
+      await this.delete(id);
+    }
+
+    return toDelete.length;
+  }
+
+  /**
+   * Clear all executions
+   */
+  clear(): void {
+    this.executions.clear();
+    this.workflowIndex.clear();
+  }
+}