flowfn 0.0.1

package/src/core/flow-fn.ts

@@ -0,0 +1,198 @@
+import { FlowAdapter } from "../adapters/base.js";
+import { MemoryAdapter } from "../adapters/memory.js";
+import { Queue, QueueOptions } from "../queue/types.js";
+import { QueueImpl } from "../queue/queue.js";
+import { Stream, StreamOptions } from "../stream/types.js";
+import { StreamImpl } from "../stream/stream.js";
+import { Workflow, WorkflowBuilder } from "../workflow/types.js";
+import { WorkflowBuilderImpl } from "../workflow/workflow.js";
+import { Scheduler } from "./scheduler.js";
+import { MetricsManager } from "./metrics.js";
+import {
+  HealthCheckerImpl,
+  HealthStatus,
+  MemoryEventTracker,
+  EventTracker,
+} from "../monitoring/health.js";
+import EventEmitter from "eventemitter3";
+
+export interface FlowFnConfig {
+  adapter:
+    | FlowAdapter
+    | "memory"
+    | "redis"
+    | "postgres"
+    | "d1"
+    | "sqs"
+    | "kafka";
+  namespace?: string;
+  defaultJobOptions?: any;
+  defaultQueueOptions?: QueueOptions;
+  defaultStreamOptions?: StreamOptions;
+  telemetry?: {
+    enabled: boolean;
+    provider?: "opentelemetry" | "custom";
+  };
+  onError?: (error: Error, context: any) => void;
+}
+
+export interface FlowFn {
+  queue<T = any>(name: string, options?: QueueOptions): Queue<T>;
+  listQueues(): Promise<string[]>;
+
+  stream<T = any>(name: string, options?: StreamOptions): Stream<T>;
+  listStreams(): Promise<string[]>;
+
+  workflow<T = any>(name: string): WorkflowBuilder<T>;
+  listWorkflows(): Promise<Workflow[]>;
+
+  scheduler(): Scheduler;
+
+  metrics: MetricsManager;
+  healthCheck(): Promise<HealthStatus>;
+  getEventTracker(): EventTracker;
+
+  on(event: string, handler: (...args: any[]) => void): void;
+  off(event: string, handler: (...args: any[]) => void): void;
+
+  close(): Promise<void>;
+}
+
+export class FlowFnImpl extends EventEmitter implements FlowFn {
+  private adapter: FlowAdapter;
+  private queues: Map<string, Queue> = new Map();
+  private streams: Map<string, Stream> = new Map();
+  private _metrics: MetricsManager;
+  private _scheduler: Scheduler;
+  private healthChecker: HealthCheckerImpl;
+  private eventTracker: EventTracker;
+  private startTime: number;
+
+  constructor(config: FlowFnConfig) {
+    super();
+    this.startTime = Date.now();
+
+    if (config.adapter === "memory") {
+      this.adapter = new MemoryAdapter();
+    } else if (typeof config.adapter === "string") {
+      // Placeholder for other string-based adapter inits
+      throw new Error(
+        `Adapter ${config.adapter} not automatically initialized yet. Pass an instance.`
+      );
+    } else {
+      this.adapter = config.adapter;
+    }
+
+    this._metrics = new MetricsManager(this.adapter);
+    this._scheduler = new Scheduler();
+    this.healthChecker = new HealthCheckerImpl();
+    this.eventTracker = new MemoryEventTracker();
+
+    // Add custom health checks
+    this.setupHealthChecks();
+  }
+
+  private setupHealthChecks(): void {
+    // Queue health check
+    this.healthChecker.addCheck("queues", async () => {
+      const queueCount = this.queues.size;
+      return {
+        name: "queues",
+        status: "pass",
+        details: {
+          count: queueCount,
+          active: Array.from(this.queues.keys()),
+        },
+      };
+    });
+
+    // Adapter health check
+    this.healthChecker.addCheck("adapter", async () => {
+      try {
+        // Simple ping check
+        await this.adapter.getQueueStats("health-check");
+        return {
+          name: "adapter",
+          status: "pass",
+          message: "Adapter responding",
+        };
+      } catch (error) {
+        return {
+          name: "adapter",
+          status: "fail",
+          message: (error as Error).message,
+        };
+      }
+    });
+  }
+
+  queue<T = any>(name: string, options?: QueueOptions): Queue<T> {
+    if (!this.queues.has(name)) {
+      this.queues.set(name, new QueueImpl<T>(name, this.adapter, options));
+    }
+    return this.queues.get(name) as Queue<T>;
+  }
+
+  async listQueues(): Promise<string[]> {
+    return Array.from(this.queues.keys());
+  }
+
+  stream<T = any>(name: string, options?: StreamOptions): Stream<T> {
+    if (!this.streams.has(name)) {
+      this.streams.set(name, new StreamImpl<T>(name, this.adapter, options));
+    }
+    return this.streams.get(name) as Stream<T>;
+  }
+
+  async listStreams(): Promise<string[]> {
+    return Array.from(this.streams.keys());
+  }
+
+  workflow<T = any>(name: string): WorkflowBuilder<T> {
+    return new WorkflowBuilderImpl<T>(name, this.adapter);
+  }
+
+  async listWorkflows(): Promise<Workflow[]> {
+    return [];
+  }
+
+  scheduler(): Scheduler {
+    return this._scheduler;
+  }
+
+  get metrics(): MetricsManager {
+    return this._metrics;
+  }
+
+  async healthCheck(): Promise<HealthStatus> {
+    const health = await this.healthChecker.check();
+
+    // Track health check event
+    this.eventTracker.track({
+      type: "health.check",
+      category: "system",
+      severity: health.healthy ? "info" : "warn",
+      message: health.healthy ? "System healthy" : "System unhealthy",
+      metadata: { checksCount: health.checks.length },
+    });
+
+    return health;
+  }
+
+  /**
+   * Get event tracker
+   */
+  getEventTracker(): EventTracker {
+    return this.eventTracker;
+  }
+
+  async close(): Promise<void> {
+    await this.adapter.cleanup();
+    for (const q of this.queues.values()) await q.close();
+    for (const s of this.streams.values()) await s.close();
+  }
+}
+
+export function createFlow(config: FlowFnConfig): FlowFn {
+  return new FlowFnImpl(config);
+}

package/src/core/metrics.ts

@@ -0,0 +1,198 @@
+import { FlowAdapter } from "../adapters/base.js";
+
+export interface MetricDataPoint {
+  timestamp: number;
+  value: number;
+  tags?: Record<string, string>;
+}
+
+export interface TimeSeriesMetrics {
+  dataPoints: MetricDataPoint[];
+
+  // Aggregations
+  min: number;
+  max: number;
+  avg: number;
+  sum: number;
+  count: number;
+  p50?: number;
+  p95?: number;
+  p99?: number;
+}
+
+export class MetricsManager {
+  private adapter: FlowAdapter;
+  private metrics: Map<string, MetricDataPoint[]> = new Map();
+  private maxDataPoints = 1000;
+
+  constructor(adapter: FlowAdapter) {
+    this.adapter = adapter;
+  }
+
+  /**
+   * Record a metric data point
+   */
+  record(name: string, value: number, tags?: Record<string, string>): void {
+    const key = this.getMetricKey(name, tags);
+    if (!this.metrics.has(key)) {
+      this.metrics.set(key, []);
+    }
+
+    const dataPoints = this.metrics.get(key)!;
+    dataPoints.push({
+      timestamp: Date.now(),
+      value,
+      tags,
+    });
+
+    // Trim if exceeds max
+    if (dataPoints.length > this.maxDataPoints) {
+      this.metrics.set(key, dataPoints.slice(-this.maxDataPoints));
+    }
+  }
+
+  /**
+   * Get time series for a metric
+   */
+  getTimeSeries(
+    name: string,
+    options?: {
+      tags?: Record<string, string>;
+      since?: number;
+      limit?: number;
+    }
+  ): TimeSeriesMetrics | null {
+    const key = this.getMetricKey(name, options?.tags);
+    let dataPoints = this.metrics.get(key) || [];
+
+    if (options?.since) {
+      dataPoints = dataPoints.filter((dp) => dp.timestamp >= options.since!);
+    }
+
+    if (options?.limit) {
+      dataPoints = dataPoints.slice(-options.limit);
+    }
+
+    if (dataPoints.length === 0) {
+      return null;
+    }
+
+    return this.calculateAggregations(dataPoints);
+  }
+
+  private calculateAggregations(
+    dataPoints: MetricDataPoint[]
+  ): TimeSeriesMetrics {
+    const values = dataPoints.map((dp) => dp.value).sort((a, b) => a - b);
+    const sum = values.reduce((a, b) => a + b, 0);
+    const count = values.length;
+
+    return {
+      dataPoints,
+      min: values[0],
+      max: values[count - 1],
+      avg: sum / count,
+      sum,
+      count,
+      p50: this.percentile(values, 50),
+      p95: this.percentile(values, 95),
+      p99: this.percentile(values, 99),
+    };
+  }
+
+  private percentile(sorted: number[], p: number): number {
+    const index = Math.ceil((sorted.length * p) / 100) - 1;
+    return sorted[Math.max(0, index)];
+  }
+
+  private getMetricKey(name: string, tags?: Record<string, string>): string {
+    if (!tags) return name;
+    const tagStr = Object.entries(tags)
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([k, v]) => `${k}:${v}`)
+      .join(",");
+    return `${name}{${tagStr}}`;
+  }
+
+  async getQueueMetrics(name: string): Promise<any> {
+    const stats = await this.adapter.getQueueStats(name);
+
+    // Get throughput from recorded metrics
+    const throughputMetrics = this.getTimeSeries("queue.throughput", {
+      tags: { queue: name },
+      since: Date.now() - 60000, // Last minute
+    });
+
+    const durationMetrics = this.getTimeSeries("queue.job.duration", {
+      tags: { queue: name },
+    });
+
+    return {
+      ...stats,
+      throughput: throughputMetrics?.avg || 0,
+      avgDuration: durationMetrics?.avg || 0,
+      p95Duration: durationMetrics?.p95 || 0,
+    };
+  }
+
+  async getStreamMetrics(name: string): Promise<any> {
+    const info = await this.adapter.getStreamInfo(name);
+
+    const throughputMetrics = this.getTimeSeries("stream.throughput", {
+      tags: { stream: name },
+      since: Date.now() - 1000, // Last second
+    });
+
+    return {
+      ...info,
+      lag: 0,
+      throughput: throughputMetrics?.avg || 0,
+      avgLatency: 0,
+    };
+  }
+
+  async getWorkflowMetrics(name: string): Promise<any> {
+    const metricsData = this.getTimeSeries("workflow.executions", {
+      tags: { workflow: name },
+    });
+
+    return {
+      totalExecutions: metricsData?.count || 0,
+      running: 0,
+      completed: 0,
+      failed: 0,
+      successRate: 0,
+      avgDuration: metricsData?.avg || 0,
+    };
+  }
+
+  async getSystemMetrics(): Promise<any> {
+    const memUsage =
+      typeof process !== "undefined" && process.memoryUsage
+        ? process.memoryUsage().heapUsed
+        : 0;
+
+    return {
+      queues: 0,
+      streams: 0,
+      workflows: 0,
+      workers: 0,
+      memoryUsage: memUsage,
+    };
+  }
+
+  /**
+   * Clear old metrics
+   */
+  cleanup(maxAge: number): void {
+    const now = Date.now();
+    for (const [key, dataPoints] of this.metrics.entries()) {
+      const filtered = dataPoints.filter((dp) => now - dp.timestamp <= maxAge);
+      if (filtered.length === 0) {
+        this.metrics.delete(key);
+      } else {
+        this.metrics.set(key, filtered);
+      }
+    }
+  }
+}

package/src/core/scheduler.test.ts

@@ -0,0 +1,80 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { Scheduler } from './scheduler.js';
+
+describe('Scheduler', () => {
+    let scheduler: Scheduler;
+
+    beforeEach(() => {
+        scheduler = new Scheduler();
+        vi.useFakeTimers();
+    });
+
+    afterEach(async () => {
+        await scheduler.close();
+        vi.useRealTimers();
+    });
+
+    it('should schedule a task using cron pattern', async () => {
+        let count = 0;
+        // Every second
+        await scheduler.schedule('test', '* * * * * *', async () => {
+            count++;
+        });
+
+        const list = await scheduler.list();
+        expect(list).toHaveLength(1);
+        expect(list[0].name).toBe('test');
+
+        await vi.advanceTimersByTimeAsync(1000);
+        expect(count).toBe(1);
+
+        await vi.advanceTimersByTimeAsync(1000);
+        expect(count).toBe(2);
+    });
+
+    it('should schedule a task using "every" interval', async () => {
+        let count = 0;
+        await scheduler.schedule('repeat', { every: 100 }, async () => {
+            count++;
+        });
+
+        await vi.advanceTimersByTimeAsync(100);
+        expect(count).toBe(1);
+
+        await vi.advanceTimersByTimeAsync(100);
+        expect(count).toBe(2);
+    });
+
+    it('should respect limits', async () => {
+        let count = 0;
+        await scheduler.schedule('limited', { every: 100, limit: 2 }, async () => {
+            count++;
+        });
+
+        await vi.advanceTimersByTimeAsync(100);
+        await vi.advanceTimersByTimeAsync(100);
+        await vi.advanceTimersByTimeAsync(100);
+        
+        expect(count).toBe(2);
+        const list = await scheduler.list();
+        expect(list).toHaveLength(0);
+    });
+
+    it('should pause and resume', async () => {
+        let count = 0;
+        await scheduler.schedule('pause-test', { every: 100 }, async () => {
+            count++;
+        });
+
+        await vi.advanceTimersByTimeAsync(100);
+        expect(count).toBe(1);
+
+        await scheduler.pause('pause-test');
+        await vi.advanceTimersByTimeAsync(100);
+        expect(count).toBe(1);
+
+        await scheduler.resume('pause-test');
+        await vi.advanceTimersByTimeAsync(100);
+        expect(count).toBe(2);
+    });
+});

package/src/core/scheduler.ts

@@ -0,0 +1,154 @@
+import parser from 'cron-parser';
+
+export interface ScheduleOptions {
+    pattern?: string;
+    timezone?: string;
+    data?: any;
+    startDate?: Date;
+    endDate?: Date;
+    limit?: number;
+    every?: number; // ms
+}
+
+export class Scheduler {
+    private tasks: Map<string, {
+        pattern: string | ScheduleOptions;
+        handler: Function;
+        nextRun: number;
+        timer?: NodeJS.Timeout;
+        count: number;
+        paused: boolean;
+    }> = new Map();
+
+    async schedule(name: string, pattern: string | ScheduleOptions, handler: Function): Promise<void> {
+        await this.cancel(name);
+        
+        const options: ScheduleOptions = typeof pattern === 'string' ? { pattern } : pattern;
+        const now = Date.now();
+        let nextRun = 0;
+
+        if (options.pattern) {
+            const interval = parser.parseExpression(options.pattern, {
+                currentDate: options.startDate || new Date(),
+                tz: options.timezone
+            });
+            nextRun = interval.next().getTime();
+        } else if (options.every) {
+            nextRun = (options.startDate?.getTime() || now) + options.every;
+        }
+
+        this.tasks.set(name, {
+            pattern,
+            handler,
+            nextRun,
+            count: 0,
+            paused: false
+        });
+
+        this.plan(name);
+    }
+    
+    async repeat(name: string, options: any, handler: Function): Promise<void> {
+        return this.schedule(name, options, handler);
+    }
+
+    private plan(name: string) {
+        const task = this.tasks.get(name);
+        if (!task || task.paused) return;
+
+        const now = Date.now();
+        const delay = Math.max(0, task.nextRun - now);
+
+        if (task.timer) clearTimeout(task.timer);
+
+        task.timer = setTimeout(async () => {
+            if (task.paused) return;
+            
+            try {
+                const options: ScheduleOptions = typeof task.pattern === 'string' ? { pattern: task.pattern } : task.pattern;
+                await task.handler(options.data);
+                task.count++;
+
+                // Check limits
+                if (options.limit && task.count >= options.limit) {
+                    this.tasks.delete(name);
+                    return;
+                }
+
+                // Plan next run
+                if (options.pattern) {
+                    const interval = parser.parseExpression(options.pattern, {
+                        currentDate: new Date(),
+                        tz: options.timezone
+                    });
+                    task.nextRun = interval.next().getTime();
+                } else if (options.every) {
+                    task.nextRun = Date.now() + options.every;
+                }
+
+                if (options.endDate && task.nextRun > options.endDate.getTime()) {
+                    this.tasks.delete(name);
+                    return;
+                }
+
+                this.plan(name);
+            } catch (err) {
+                console.error(`Scheduler task "${name}" failed:`, err);
+                // Still plan next run? Usually yes.
+                this.plan(name);
+            }
+        }, delay);
+    }
+
+    async list(): Promise<any[]> {
+        return Array.from(this.tasks.entries()).map(([name, task]) => ({
+            name,
+            nextRun: new Date(task.nextRun),
+            count: task.count,
+            paused: task.paused
+        }));
+    }
+
+    async cancel(name: string): Promise<void> {
+        const task = this.tasks.get(name);
+        if (task) {
+            if (task.timer) clearTimeout(task.timer);
+            this.tasks.delete(name);
+        }
+    }
+
+    async pause(name: string): Promise<void> {
+        const task = this.tasks.get(name);
+        if (task) {
+            task.paused = true;
+            if (task.timer) clearTimeout(task.timer);
+        }
+    }
+
+    async resume(name: string): Promise<void> {
+        const task = this.tasks.get(name);
+        if (task && task.paused) {
+            task.paused = false;
+            // Recalculate next run if it passed while paused?
+            // For cron, we just find the next occurrence from now.
+            const options: ScheduleOptions = typeof task.pattern === 'string' ? { pattern: task.pattern } : task.pattern;
+            if (options.pattern) {
+                const interval = parser.parseExpression(options.pattern, {
+                    currentDate: new Date(),
+                    tz: options.timezone
+                });
+                task.nextRun = interval.next().getTime();
+            } else if (options.every) {
+                task.nextRun = Date.now() + options.every;
+            }
+            this.plan(name);
+        }
+    }
+
+    async close(): Promise<void> {
+        for (const task of this.tasks.values()) {
+            if (task.timer) clearTimeout(task.timer);
+        }
+        this.tasks.clear();
+    }
+}

package/src/index.ts

@@ -0,0 +1,57 @@
+export * from "./core/flow-fn.js";
+export * from "./core/scheduler.js";
+export * from "./core/metrics.js";
+
+export * from "./queue/types.js";
+export * from "./queue/worker.js";
+export * from "./queue/dlq.js";
+export * from "./stream/types.js";
+export * from "./workflow/types.js";
+
+export * from "./adapters/base.js";
+export * from "./adapters/memory.js";
+export * from "./adapters/redis.js";
+export * from "./adapters/postgres/index.js";
+
+export * from "./patterns/backoff.js";
+export * from "./patterns/circuit-breaker.js";
+export * from "./patterns/rate-limit.js";
+export {
+  batch,
+  chunk,
+  processBatches,
+  BatchWriter,
+  batchByKey,
+  BatchAccumulator,
+  type BatchOptions as BatchingOptions,
+} from "./patterns/batching.js";
+export * from "./patterns/priority.js";
+
+export * from "./monitoring/health.js";
+
+export * from "./utils/hashing.js";
+export * from "./utils/id-generator.js";
+export {
+  toMilliseconds,
+  fromMilliseconds,
+  formatDuration,
+  sleep,
+  sleepDuration,
+  timeout,
+  now,
+  delayUntil,
+  isPast,
+  isFuture,
+  addDuration,
+  parseDuration,
+  type Duration as TimeDuration,
+} from "./utils/time.js";
+export * from "./utils/serialization.js";
+
+export * from "./storage/job-storage.js";
+export {
+  MemoryWorkflowStorage,
+  WorkflowStorage,
+  type ListOptions as StorageListOptions,
+} from "./storage/workflow-storage.js";
+export * from "./storage/event-log.js";