flowfn 0.0.1

package/src/monitoring/health.ts

@@ -0,0 +1,261 @@
+/**
+ * Monitoring and health check system for FlowFn
+ */
+
+export interface HealthStatus {
+  healthy: boolean;
+  timestamp: number;
+  checks: HealthCheck[];
+  details?: Record<string, any>;
+}
+
+export interface HealthCheck {
+  name: string;
+  status: "pass" | "fail" | "warn";
+  message?: string;
+  responseTime?: number;
+  details?: Record<string, any>;
+}
+
+export interface MonitoringMetrics {
+  // Queue metrics
+  queues?: {
+    [queueName: string]: {
+      waiting: number;
+      active: number;
+      completed: number;
+      failed: number;
+      delayed: number;
+      throughput: number; // jobs/min
+      avgDuration: number; // ms
+      errorRate: number; // %
+    };
+  };
+
+  // Stream metrics
+  streams?: {
+    [streamName: string]: {
+      messageCount: number;
+      publishRate: number; // msgs/sec
+      consumerCount: number;
+      lag: number;
+    };
+  };
+
+  // Workflow metrics
+  workflows?: {
+    [workflowName: string]: {
+      totalExecutions: number;
+      running: number;
+      completed: number;
+      failed: number;
+      successRate: number;
+      avgDuration: number;
+    };
+  };
+
+  // System metrics
+  system?: {
+    uptime: number;
+    memoryUsage: number;
+    cpuUsage?: number;
+  };
+}
+
+export interface HealthChecker {
+  /**
+   * Perform health check
+   */
+  check(): Promise<HealthStatus>;
+
+  /**
+   * Add custom health check
+   */
+  addCheck(name: string, checker: () => Promise<HealthCheck>): void;
+
+  /**
+   * Remove health check
+   */
+  removeCheck(name: string): void;
+}
+
+/**
+ * Health checker implementation
+ */
+export class HealthCheckerImpl implements HealthChecker {
+  private checks: Map<string, () => Promise<HealthCheck>> = new Map();
+  private startTime = Date.now();
+
+  constructor() {
+    // Add default system checks
+    this.addCheck("uptime", async () => ({
+      name: "uptime",
+      status: "pass",
+      responseTime: 0,
+      details: {
+        uptime: Date.now() - this.startTime,
+        startTime: this.startTime,
+      },
+    }));
+
+    this.addCheck("memory", async () => {
+      if (typeof process !== "undefined" && process.memoryUsage) {
+        const mem = process.memoryUsage();
+        const usedMB = mem.heapUsed / 1024 / 1024;
+        const totalMB = mem.heapTotal / 1024 / 1024;
+        const usagePercent = (usedMB / totalMB) * 100;
+
+        return {
+          name: "memory",
+          status: usagePercent > 90 ? "warn" : "pass",
+          message: usagePercent > 90 ? "High memory usage" : undefined,
+          responseTime: 0,
+          details: {
+            heapUsedMB: Math.round(usedMB),
+            heapTotalMB: Math.round(totalMB),
+            usagePercent: Math.round(usagePercent),
+          },
+        };
+      }
+
+      return {
+        name: "memory",
+        status: "pass",
+        responseTime: 0,
+      };
+    });
+  }
+
+  addCheck(name: string, checker: () => Promise<HealthCheck>): void {
+    this.checks.set(name, checker);
+  }
+
+  removeCheck(name: string): void {
+    this.checks.delete(name);
+  }
+
+  async check(): Promise<HealthStatus> {
+    const results: HealthCheck[] = [];
+    let allHealthy = true;
+
+    for (const [name, checker] of this.checks.entries()) {
+      try {
+        const start = Date.now();
+        const result = await checker();
+        result.responseTime = Date.now() - start;
+        results.push(result);
+
+        if (result.status === "fail") {
+          allHealthy = false;
+        }
+      } catch (error) {
+        results.push({
+          name,
+          status: "fail",
+          message: (error as Error).message,
+          responseTime: 0,
+        });
+        allHealthy = false;
+      }
+    }
+
+    return {
+      healthy: allHealthy,
+      timestamp: Date.now(),
+      checks: results,
+    };
+  }
+}
+
+/**
+ * Event tracking for monitoring
+ */
+export interface TrackedEvent {
+  id: string;
+  type: string;
+  category: "queue" | "stream" | "workflow" | "system";
+  severity: "info" | "warn" | "error";
+  message: string;
+  timestamp: number;
+  metadata?: Record<string, any>;
+}
+
+export interface EventTracker {
+  /**
+   * Track an event
+   */
+  track(event: Omit<TrackedEvent, "id" | "timestamp">): void;
+
+  /**
+   * Get events
+   */
+  getEvents(filter?: {
+    category?: string;
+    severity?: string;
+    since?: number;
+    limit?: number;
+  }): TrackedEvent[];
+
+  /**
+   * Clear old events
+   */
+  cleanup(maxAge: number): number;
+}
+
+export class MemoryEventTracker implements EventTracker {
+  private events: TrackedEvent[] = [];
+  private maxEvents = 10000;
+
+  track(event: Omit<TrackedEvent, "id" | "timestamp">): void {
+    const trackedEvent: TrackedEvent = {
+      ...event,
+      id: `evt_${Date.now()}_${Math.random().toString(36).substring(7)}`,
+      timestamp: Date.now(),
+    };
+
+    this.events.push(trackedEvent);
+
+    // Trim if exceeds max
+    if (this.events.length > this.maxEvents) {
+      this.events = this.events.slice(-this.maxEvents);
+    }
+  }
+
+  getEvents(filter?: {
+    category?: string;
+    severity?: string;
+    since?: number;
+    limit?: number;
+  }): TrackedEvent[] {
+    let filtered = [...this.events];
+
+    if (filter?.category) {
+      filtered = filtered.filter((e) => e.category === filter.category);
+    }
+
+    if (filter?.severity) {
+      filtered = filtered.filter((e) => e.severity === filter.severity);
+    }
+
+    if (filter?.since !== undefined) {
+      filtered = filtered.filter((e) => e.timestamp >= filter.since!);
+    }
+
+    if (filter?.limit) {
+      filtered = filtered.slice(-filter.limit);
+    }
+
+    return filtered;
+  }
+
+  cleanup(maxAge: number): number {
+    const now = Date.now();
+    const before = this.events.length;
+    this.events = this.events.filter((e) => now - e.timestamp <= maxAge);
+    return before - this.events.length;
+  }
+
+  clear(): void {
+    this.events = [];
+  }
+}

package/src/patterns/backoff.ts

@@ -0,0 +1,30 @@
+import { BackoffOptions } from '../queue/types.js';
+
+export function calculateBackoff(attemptsMade: number, options: BackoffOptions): number {
+  const { type, delay, maxDelay } = options;
+
+  let resultDelay: number;
+
+  switch (type) {
+    case 'fixed':
+      resultDelay = delay;
+      break;
+    case 'exponential':
+      resultDelay = delay * Math.pow(2, attemptsMade - 1);
+      break;
+    case 'custom':
+      // Custom should probably be a function passed in opts, 
+      // but for serialization we might need a registry.
+      // Default to fixed for now if not handled.
+      resultDelay = delay;
+      break;
+    default:
+      resultDelay = delay;
+  }
+
+  if (maxDelay && resultDelay > maxDelay) {
+    return maxDelay;
+  }
+
+  return resultDelay;
+}

package/src/patterns/batching.ts

@@ -0,0 +1,248 @@
+/**
+ * Batching utilities for FlowFn
+ */
+
+export interface BatchOptions {
+  /**
+   * Maximum number of items per batch
+   */
+  maxSize: number;
+
+  /**
+   * Maximum time to wait before flushing batch (ms)
+   */
+  maxWait?: number;
+
+  /**
+   * Minimum size before processing (default: 1)
+   */
+  minSize?: number;
+}
+
+export interface BatchProcessor<T, R> {
+  /**
+   * Process a batch of items
+   */
+  (items: T[]): Promise<R[]>;
+}
+
+/**
+ * Batch accumulator for collecting and processing items in batches
+ */
+export class BatchAccumulator<T, R = any> {
+  private batch: T[] = [];
+  private timer: NodeJS.Timeout | null = null;
+  private options: Required<BatchOptions>;
+  private processor: BatchProcessor<T, R>;
+  private pending: Array<{
+    resolve: (value: R) => void;
+    reject: (error: Error) => void;
+  }> = [];
+
+  constructor(processor: BatchProcessor<T, R>, options: BatchOptions) {
+    this.processor = processor;
+    this.options = {
+      minSize: 1,
+      maxWait: 1000,
+      ...options,
+    };
+  }
+
+  /**
+   * Add an item to the batch
+   */
+  async add(item: T): Promise<R> {
+    return new Promise<R>((resolve, reject) => {
+      this.batch.push(item);
+      this.pending.push({ resolve, reject });
+
+      // Flush if batch is full
+      if (this.batch.length >= this.options.maxSize) {
+        this.flush();
+      } else if (!this.timer) {
+        // Set timer for max wait
+        this.timer = setTimeout(() => {
+          this.flush();
+        }, this.options.maxWait);
+      }
+    });
+  }
+
+  /**
+   * Manually flush the current batch
+   */
+  async flush(): Promise<void> {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+
+    if (this.batch.length < this.options.minSize) {
+      return;
+    }
+
+    const currentBatch = this.batch;
+    const currentPending = this.pending;
+
+    this.batch = [];
+    this.pending = [];
+
+    try {
+      const results = await this.processor(currentBatch);
+
+      // Resolve all pending promises
+      for (let i = 0; i < currentPending.length; i++) {
+        currentPending[i].resolve(results[i]);
+      }
+    } catch (error) {
+      // Reject all pending promises
+      for (const pending of currentPending) {
+        pending.reject(error as Error);
+      }
+    }
+  }
+
+  /**
+   * Get current batch size
+   */
+  size(): number {
+    return this.batch.length;
+  }
+
+  /**
+   * Clear the batch without processing
+   */
+  clear(): void {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+    this.batch = [];
+
+    // Reject all pending
+    for (const pending of this.pending) {
+      pending.reject(new Error("Batch cleared"));
+    }
+    this.pending = [];
+  }
+}
+
+/**
+ * Create a batched version of a function
+ */
+export function batch<T, R>(
+  fn: (items: T[]) => Promise<R[]>,
+  options: BatchOptions
+): (item: T) => Promise<R> {
+  const accumulator = new BatchAccumulator(fn, options);
+  return (item: T) => accumulator.add(item);
+}
+
+/**
+ * Batch array into chunks
+ */
+export function chunk<T>(array: T[], size: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += size) {
+    chunks.push(array.slice(i, i + size));
+  }
+  return chunks;
+}
+
+/**
+ * Process array in batches with delay between batches
+ */
+export async function processBatches<T, R>(
+  items: T[],
+  processor: (batch: T[]) => Promise<R[]>,
+  options: { batchSize: number; delayMs?: number }
+): Promise<R[]> {
+  const batches = chunk(items, options.batchSize);
+  const results: R[] = [];
+
+  for (let i = 0; i < batches.length; i++) {
+    const batchResults = await processor(batches[i]);
+    results.push(...batchResults);
+
+    // Add delay between batches (except after last batch)
+    if (i < batches.length - 1 && options.delayMs) {
+      await new Promise((resolve) => setTimeout(resolve, options.delayMs));
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Batch write operations with auto-flush
+ */
+export class BatchWriter<T> {
+  private accumulator: BatchAccumulator<T, void>;
+
+  constructor(writer: (items: T[]) => Promise<void>, options: BatchOptions) {
+    this.accumulator = new BatchAccumulator(async (items) => {
+      await writer(items);
+      return new Array(items.length).fill(undefined);
+    }, options);
+  }
+
+  async write(item: T): Promise<void> {
+    return this.accumulator.add(item);
+  }
+
+  async flush(): Promise<void> {
+    return this.accumulator.flush();
+  }
+
+  size(): number {
+    return this.accumulator.size();
+  }
+
+  clear(): void {
+    this.accumulator.clear();
+  }
+}
+
+/**
+ * Group items by a key function and process in batches
+ */
+export async function batchByKey<T, R>(
+  items: T[],
+  keyFn: (item: T) => string,
+  processor: (key: string, items: T[]) => Promise<R[]>,
+  options?: { concurrency?: number }
+): Promise<R[]> {
+  const grouped = new Map<string, T[]>();
+
+  // Group items by key
+  for (const item of items) {
+    const key = keyFn(item);
+    if (!grouped.has(key)) {
+      grouped.set(key, []);
+    }
+    grouped.get(key)!.push(item);
+  }
+
+  // Process each group
+  const results: R[] = [];
+  const entries = Array.from(grouped.entries());
+
+  if (options?.concurrency) {
+    // Process with concurrency limit
+    for (let i = 0; i < entries.length; i += options.concurrency) {
+      const batch = entries.slice(i, i + options.concurrency);
+      const batchResults = await Promise.all(
+        batch.map(([key, items]) => processor(key, items))
+      );
+      results.push(...batchResults.flat());
+    }
+  } else {
+    // Process sequentially
+    for (const [key, keyItems] of entries) {
+      const batchResults = await processor(key, keyItems);
+      results.push(...batchResults);
+    }
+  }
+
+  return results;
+}

package/src/patterns/circuit-breaker.test.ts

@@ -0,0 +1,52 @@
+import { describe, it, expect, vi } from 'vitest';
+import { circuitBreaker } from './circuit-breaker.js';
+
+describe('Circuit Breaker', () => {
+    it('should open after threshold is reached', async () => {
+        let calls = 0;
+        const handler = async () => {
+            calls++;
+            throw new Error('fail');
+        };
+
+        const cb = circuitBreaker({ threshold: 2, timeout: 1000 }, handler);
+
+        // First failure
+        await expect(cb({} as any)).rejects.toThrow('fail');
+        expect(calls).toBe(1);
+
+        // Second failure -> Should open
+        await expect(cb({} as any)).rejects.toThrow('fail');
+        expect(calls).toBe(2);
+
+        // Third call -> Should be rejected immediately without calling handler
+        await expect(cb({} as any)).rejects.toThrow('Circuit Breaker is OPEN');
+        expect(calls).toBe(2);
+    });
+
+    it('should transition to half-open after timeout', async () => {
+        vi.useFakeTimers();
+        let calls = 0;
+        const handler = async () => {
+            calls++;
+            if (calls > 2) return 'success';
+            throw new Error('fail');
+        };
+
+        const cb = circuitBreaker({ threshold: 2, timeout: 1000 }, handler);
+
+        await expect(cb({} as any)).rejects.toThrow('fail');
+        await expect(cb({} as any)).rejects.toThrow('fail');
+        await expect(cb({} as any)).rejects.toThrow('Circuit Breaker is OPEN');
+
+        // Fast forward
+        await vi.advanceTimersByTimeAsync(1001);
+
+        // Should call handler again (half-open)
+        const result = await cb({} as any);
+        expect(result).toBe('success');
+        expect(calls).toBe(3);
+
+        vi.useRealTimers();
+    });
+});

package/src/patterns/circuit-breaker.ts

@@ -0,0 +1,52 @@
+import { Job } from '../queue/types.js';
+
+export interface CircuitBreakerOptions {
+  threshold: number;
+  timeout: number;
+  resetTimeout?: number;
+}
+
+export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
+
+export function circuitBreaker<T>(
+  options: CircuitBreakerOptions,
+  handler: (job: Job<T>) => Promise<any>
+) {
+  let state: CircuitState = 'CLOSED';
+  let failures = 0;
+  let lastFailureTime = 0;
+  let lastSuccessTime = 0;
+
+  return async (job: Job<T>) => {
+    const now = Date.now();
+
+    if (state === 'OPEN') {
+      if (now - lastFailureTime > options.timeout) {
+        state = 'HALF_OPEN';
+      } else {
+        throw new Error('Circuit Breaker is OPEN');
+      }
+    }
+
+    try {
+      const result = await handler(job);
+      
+      if (state === 'HALF_OPEN') {
+        state = 'CLOSED';
+        failures = 0;
+      }
+      
+      lastSuccessTime = now;
+      return result;
+    } catch (err) {
+      failures++;
+      lastFailureTime = now;
+
+      if (state === 'HALF_OPEN' || failures >= options.threshold) {
+        state = 'OPEN';
+      }
+
+      throw err;
+    }
+  };
+}