flowfn 0.0.1

package/src/queue/dlq.ts

@@ -0,0 +1,222 @@
+/**
+ * Dead-Letter Queue (DLQ) implementation for FlowFn
+ */
+
+import { Job, JobOptions } from "../queue/types.js";
+
+export interface DLQOptions {
+  /**
+   * Queue name for dead-letter jobs
+   */
+  queueName?: string;
+
+  /**
+   * Maximum retries before moving to DLQ
+   */
+  maxRetries?: number;
+
+  /**
+   * Time to live for DLQ items (ms)
+   */
+  ttl?: number;
+
+  /**
+   * Handler called when job moves to DLQ
+   */
+  onDLQ?: (job: Job, reason: string) => void | Promise<void>;
+}
+
+export interface DLQJob<T = any> extends Job<T> {
+  /**
+   * Original queue name
+   */
+  originalQueue: string;
+
+  /**
+   * Reason for DLQ
+   */
+  dlqReason: string;
+
+  /**
+   * Timestamp when moved to DLQ
+   */
+  dlqTimestamp: number;
+
+  /**
+   * All error messages from attempts
+   */
+  errors: string[];
+}
+
+export interface DLQManager {
+  /**
+   * Move job to DLQ
+   */
+  moveToDLQ<T>(job: Job<T>, reason: string): Promise<DLQJob<T>>;
+
+  /**
+   * Get all DLQ jobs
+   */
+  getAll(): Promise<DLQJob[]>;
+
+  /**
+   * Get DLQ jobs by original queue
+   */
+  getByQueue(queueName: string): Promise<DLQJob[]>;
+
+  /**
+   * Retry a DLQ job
+   */
+  retry<T>(jobId: string): Promise<Job<T>>;
+
+  /**
+   * Retry all DLQ jobs from a queue
+   */
+  retryAll(queueName: string): Promise<number>;
+
+  /**
+   * Delete a DLQ job
+   */
+  delete(jobId: string): Promise<void>;
+
+  /**
+   * Clean expired DLQ jobs
+   */
+  clean(maxAge: number): Promise<number>;
+
+  /**
+   * Get DLQ stats
+   */
+  getStats(): Promise<{ total: number; byQueue: Record<string, number> }>;
+}
+
+/**
+ * In-memory DLQ implementation
+ */
+export class MemoryDLQManager implements DLQManager {
+  private dlqJobs: Map<string, DLQJob> = new Map();
+  private options: DLQOptions;
+
+  constructor(options: DLQOptions = {}) {
+    this.options = {
+      queueName: "dlq",
+      maxRetries: 3,
+      ttl: 7 * 24 * 60 * 60 * 1000, // 7 days
+      ...options,
+    };
+  }
+
+  async moveToDLQ<T>(job: Job<T>, reason: string): Promise<DLQJob<T>> {
+    const dlqJob: DLQJob<T> = {
+      ...job,
+      originalQueue: job.name,
+      dlqReason: reason,
+      dlqTimestamp: Date.now(),
+      errors: [...(job.stacktrace || []), job.failedReason || reason].filter(
+        Boolean
+      ),
+    };
+
+    this.dlqJobs.set(job.id, dlqJob);
+
+    // Call handler if provided
+    if (this.options.onDLQ) {
+      await Promise.resolve(this.options.onDLQ(job, reason));
+    }
+
+    return dlqJob;
+  }
+
+  async getAll(): Promise<DLQJob[]> {
+    return Array.from(this.dlqJobs.values());
+  }
+
+  async getByQueue(queueName: string): Promise<DLQJob[]> {
+    return Array.from(this.dlqJobs.values()).filter(
+      (job) => job.originalQueue === queueName
+    );
+  }
+
+  async retry<T>(jobId: string): Promise<Job<T>> {
+    const dlqJob = this.dlqJobs.get(jobId);
+    if (!dlqJob) {
+      throw new Error(`DLQ job ${jobId} not found`);
+    }
+
+    // Remove from DLQ
+    this.dlqJobs.delete(jobId);
+
+    // Reset job state for retry
+    const retriedJob: Job<T> = {
+      ...dlqJob,
+      state: "waiting",
+      attemptsMade: 0,
+      failedReason: undefined,
+      stacktrace: [],
+      processedOn: undefined,
+      finishedOn: undefined,
+    } as Job<T>;
+
+    return retriedJob;
+  }
+
+  async retryAll(queueName: string): Promise<number> {
+    const jobs = await this.getByQueue(queueName);
+    let count = 0;
+
+    for (const job of jobs) {
+      try {
+        await this.retry(job.id);
+        count++;
+      } catch (err) {
+        console.error(`Failed to retry job ${job.id}:`, err);
+      }
+    }
+
+    return count;
+  }
+
+  async delete(jobId: string): Promise<void> {
+    this.dlqJobs.delete(jobId);
+  }
+
+  async clean(maxAge: number): Promise<number> {
+    const now = Date.now();
+    const toDelete: string[] = [];
+
+    for (const [id, job] of this.dlqJobs.entries()) {
+      if (now - job.dlqTimestamp > maxAge) {
+        toDelete.push(id);
+      }
+    }
+
+    for (const id of toDelete) {
+      this.dlqJobs.delete(id);
+    }
+
+    return toDelete.length;
+  }
+
+  async getStats(): Promise<{
+    total: number;
+    byQueue: Record<string, number>;
+  }> {
+    const byQueue: Record<string, number> = {};
+
+    for (const job of this.dlqJobs.values()) {
+      byQueue[job.originalQueue] = (byQueue[job.originalQueue] || 0) + 1;
+    }
+
+    return {
+      total: this.dlqJobs.size,
+      byQueue,
+    };
+  }
+
+  /**
+   * Clear all DLQ jobs (for testing)
+   */
+  clear(): void {
+    this.dlqJobs.clear();
+  }
+}

package/src/queue/job.ts

@@ -0,0 +1,67 @@
+import { Job, JobOptions, JobStatus } from './types.js';
+import { v4 as uuidv4 } from 'uuid';
+
+export class JobImpl<T = any> implements Job<T> {
+  id: string;
+  name: string;
+  data: T;
+  opts: JobOptions;
+  
+  state: JobStatus = 'waiting';
+  progress: number = 0;
+  returnvalue?: any;
+  
+  timestamp: number;
+  processedOn?: number;
+  finishedOn?: number;
+  delay: number = 0;
+  
+  attemptsMade: number = 0;
+  failedReason?: string;
+  stacktrace?: string[];
+
+  constructor(name: string, data: T, opts?: JobOptions) {
+      this.id = opts?.jobId || uuidv4();
+      this.name = name;
+      this.data = data;
+      this.opts = opts || {};
+      this.timestamp = Date.now();
+  }
+  
+  async update(data: Partial<T>): Promise<void> {
+    this.data = { ...this.data, ...data };
+  }
+
+  async log(message: string): Promise<void> {
+    // console.log(`[Job ${this.id}] ${message}`);
+  }
+
+  async updateProgress(progress: number): Promise<void> {
+    this.progress = progress;
+  }
+
+  async moveToCompleted(returnValue: any): Promise<void> {
+    this.state = 'completed';
+    this.returnvalue = returnValue;
+    this.finishedOn = Date.now();
+  }
+
+  async moveToFailed(error: Error): Promise<void> {
+    this.state = 'failed';
+    this.failedReason = error.message;
+    this.finishedOn = Date.now();
+  }
+
+  async retry(): Promise<void> {
+     // logical retry
+  }
+
+  async discard(): Promise<void> {
+      // discard
+  }
+
+  async waitUntilFinished(): Promise<any> {
+      // Poll or wait for event
+      return Promise.resolve();
+  }
+}

package/src/queue/queue.ts

@@ -0,0 +1,243 @@
+import {
+  Queue,
+  Job,
+  JobOptions,
+  QueueOptions,
+  JobHandler,
+  BatchHandler,
+  BatchOptions,
+  QueueStats,
+  JobStatus,
+} from "./types.js";
+import { FlowAdapter } from "../adapters/base.js";
+import { JobImpl } from "./job.js";
+import EventEmitter from "eventemitter3";
+import { calculateBackoff } from "../patterns/backoff.js";
+
+export class QueueImpl<T = any> extends EventEmitter implements Queue<T> {
+  name: string;
+  private adapter: FlowAdapter;
+  private options: QueueOptions;
+  private isProcessing = false;
+  private currentWorkers: Set<Promise<void>> = new Set();
+
+  constructor(name: string, adapter: FlowAdapter, options: QueueOptions = {}) {
+    super();
+    this.name = name;
+    this.adapter = adapter;
+    this.options = options;
+  }
+
+  async add(name: string, data: T, opts?: JobOptions): Promise<Job<T>> {
+    const job = new JobImpl<T>(name, data, {
+      ...this.options.defaultJobOptions,
+      ...opts,
+    });
+    await this.adapter.enqueue(this.name, job);
+    this.emit("waiting", job);
+    return job;
+  }
+
+  async addBulk(
+    jobs: Array<{ name: string; data: T; opts?: JobOptions }>
+  ): Promise<Job<T>[]> {
+    return Promise.all(jobs.map((j) => this.add(j.name, j.data, j.opts)));
+  }
+
+  process(arg1: any, arg2?: any, arg3?: any): void {
+    let handler: JobHandler<T>;
+    let concurrency = 1;
+
+    if (typeof arg1 === "number") {
+      concurrency = arg1;
+      handler = arg2;
+    } else if (typeof arg1 === "string") {
+      // name based processing, ignore name for now in simple impl
+      if (typeof arg2 === "number") {
+        concurrency = arg2;
+        handler = arg3;
+      } else {
+        handler = arg2;
+      }
+    } else {
+      handler = arg1;
+    }
+
+    this.resume();
+    for (let i = 0; i < concurrency; i++) {
+      const worker = this.startProcessing(handler);
+      this.currentWorkers.add(worker);
+      worker.finally(() => this.currentWorkers.delete(worker));
+    }
+  }
+
+  private async startProcessing(handler: JobHandler<T>) {
+    while (this.isProcessing) {
+      const jobData = await this.adapter.dequeue(this.name);
+      if (jobData) {
+        const job = Object.assign(
+          new JobImpl(jobData.name, jobData.data, jobData.opts),
+          jobData
+        );
+
+        // Check dependencies
+        if (job.opts.waitFor && job.opts.waitFor.length > 0) {
+          let allDone = true;
+          for (const depId of job.opts.waitFor) {
+            const depJob = await this.adapter.getJob(this.name, depId);
+            if (!depJob || depJob.state !== "completed") {
+              allDone = false;
+              break;
+            }
+          }
+
+          if (!allDone) {
+            // Not ready, requeue
+            await this.adapter.enqueue(this.name, job);
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            continue;
+          }
+        }
+
+        try {
+          job.state = "active";
+          job.processedOn = Date.now();
+          this.emit("active", job);
+
+          const result = await handler(job);
+
+          await job.moveToCompleted(result);
+          await this.adapter.ack(this.name, job.id);
+          this.emit("completed", job, result);
+        } catch (err: any) {
+          job.attemptsMade++;
+          job.stacktrace = job.stacktrace || [];
+          job.stacktrace.push(err.stack);
+
+          const maxAttempts = job.opts.attempts || 1;
+          if (job.attemptsMade < maxAttempts) {
+            job.state = "delayed";
+            const backoff = job.opts.backoff
+              ? calculateBackoff(job.attemptsMade, job.opts.backoff)
+              : 0;
+            job.opts.delay = backoff;
+            await this.adapter.enqueue(this.name, job);
+            this.emit("failed", job, err); // failed but retrying
+          } else {
+            await job.moveToFailed(err);
+            // In real impl, move to DLQ or just ack to remove from active
+            await this.adapter.ack(this.name, job.id);
+            this.emit("failed", job, err);
+          }
+        }
+      } else {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+      }
+    }
+  }
+
+  processBatch(
+    arg1: string | any,
+    arg2: number | BatchOptions | any,
+    arg3?: BatchHandler<T>
+  ): void {
+    let handler: BatchHandler<T>;
+    let batchSize = 10;
+    let maxWait = 1000;
+
+    if (typeof arg2 === "function") {
+      handler = arg2;
+    } else {
+      handler = arg3!;
+      if (typeof arg2 === "number") {
+        batchSize = arg2;
+      } else {
+        batchSize = arg2.batchSize;
+        maxWait = arg2.maxWait || 1000;
+      }
+    }
+
+    this.resume();
+    this.startBatchProcessing(handler, batchSize, maxWait);
+  }
+
+  private async startBatchProcessing(
+    handler: BatchHandler<T>,
+    batchSize: number,
+    maxWait: number
+  ) {
+    while (this.isProcessing) {
+      const batch: Job[] = [];
+      const start = Date.now();
+
+      while (batch.length < batchSize && Date.now() - start < maxWait) {
+        const jobData = await this.adapter.dequeue(this.name);
+        if (jobData) {
+          const job = Object.assign(
+            new JobImpl(jobData.name, jobData.data, jobData.opts),
+            jobData
+          );
+          batch.push(job);
+        } else {
+          await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+      }
+
+      if (batch.length > 0) {
+        try {
+          for (const job of batch) {
+            (job as any).state = "active";
+            (job as any).processedOn = Date.now();
+          }
+
+          const results = await handler(batch);
+
+          for (let i = 0; i < batch.length; i++) {
+            await batch[i].moveToCompleted(results[i]);
+            await this.adapter.ack(this.name, batch[i].id);
+          }
+        } catch (err) {
+          for (const job of batch) {
+            await job.moveToFailed(err as Error);
+            await this.adapter.ack(this.name, job.id);
+          }
+        }
+      }
+    }
+  }
+
+  async pause(): Promise<void> {
+    this.isProcessing = false;
+  }
+
+  async resume(): Promise<void> {
+    this.isProcessing = true;
+  }
+
+  async drain(): Promise<void> {
+    await Promise.all(this.currentWorkers);
+  }
+
+  async clean(grace: number, status: JobStatus): Promise<number> {
+    return this.adapter.cleanJobs(this.name, grace, status);
+  }
+
+  async getJob(jobId: string): Promise<Job<T> | null> {
+    const job = await this.adapter.getJob(this.name, jobId);
+    return job as Job<T> | null;
+  }
+
+  async getJobs(status: JobStatus): Promise<Job<T>[]> {
+    const jobs = await this.adapter.getJobs(this.name, status);
+    return jobs as Job<T>[];
+  }
+
+  async getJobCounts(): Promise<QueueStats> {
+    return this.adapter.getQueueStats(this.name);
+  }
+
+  async close(): Promise<void> {
+    this.isProcessing = false;
+    await this.drain();
+  }
+}

package/src/queue/types.ts

@@ -0,0 +1,153 @@
+export interface Queue<T = any> {
+  name: string;
+
+  // Adding jobs
+  add(name: string, data: T, opts?: JobOptions): Promise<Job<T>>;
+  addBulk(
+    jobs: Array<{ name: string; data: T; opts?: JobOptions }>
+  ): Promise<Job<T>[]>;
+
+  // Processing
+  process(handler: JobHandler<T>): void;
+  process(name: string, handler: JobHandler<T>): void;
+  process(concurrency: number, handler: JobHandler<T>): void;
+  process(name: string, concurrency: number, handler: JobHandler<T>): void;
+
+  // Batch processing
+  processBatch(name: string, batchSize: number, handler: BatchHandler<T>): void;
+  processBatch(
+    name: string,
+    options: BatchOptions,
+    handler: BatchHandler<T>
+  ): void;
+
+  // Management
+  pause(): Promise<void>;
+  resume(): Promise<void>;
+  drain(): Promise<void>;
+  clean(grace: number, status: JobStatus): Promise<number>;
+
+  // DLQ methods
+  getDLQ?(): any; // Returns DLQManager if configured
+
+  // Queries
+  getJob(jobId: string): Promise<Job<T> | null>;
+  getJobs(status: JobStatus): Promise<Job<T>[]>;
+  getJobCounts(): Promise<QueueStats>;
+
+  // Events
+  on(event: QueueEvent, handler: (...args: any[]) => void): void;
+
+  // Lifecycle
+  close(): Promise<void>;
+}
+
+export type QueueEvent =
+  | "completed"
+  | "failed"
+  | "stalled"
+  | "progress"
+  | "waiting"
+  | "active";
+
+export interface Job<T = any> {
+  id: string;
+  name: string;
+  data: T;
+  opts: JobOptions;
+
+  // Status
+  state: JobStatus;
+  progress: number;
+  returnvalue?: any;
+
+  // Timing
+  timestamp: number;
+  processedOn?: number;
+  finishedOn?: number;
+  delay: number;
+
+  // Retries
+  attemptsMade: number;
+  failedReason?: string;
+  stacktrace?: string[];
+
+  // Methods
+  update(data: Partial<T>): Promise<void>;
+  log(message: string): Promise<void>;
+  updateProgress(progress: number): Promise<void>;
+  moveToCompleted(returnValue: any): Promise<void>;
+  moveToFailed(error: Error): Promise<void>;
+  retry(): Promise<void>;
+  discard(): Promise<void>;
+  waitUntilFinished(): Promise<any>;
+}
+
+export type JobStatus =
+  | "waiting"
+  | "active"
+  | "completed"
+  | "failed"
+  | "delayed"
+  | "paused";
+
+export interface JobOptions {
+  priority?: number;
+  delay?: number;
+  attempts?: number;
+  backoff?: BackoffOptions;
+  lifo?: boolean;
+  timeout?: number;
+  removeOnComplete?: boolean | number;
+  removeOnFail?: boolean | number;
+  stackTraceLimit?: number;
+  jobId?: string;
+  preventDuplicates?: boolean;
+  deduplicationKey?: string;
+  waitFor?: string[]; // Job IDs dependency
+}
+
+export interface BackoffOptions {
+  type: "fixed" | "exponential" | "custom";
+  delay: number;
+  maxDelay?: number;
+}
+
+export interface QueueOptions {
+  defaultJobOptions?: JobOptions;
+  limiter?: RateLimiterOptions;
+  cache?: CacheOptions;
+  dlq?: {
+    enabled: boolean;
+    maxRetries?: number;
+    queueName?: string;
+  };
+}
+
+export interface RateLimiterOptions {
+  max: number;
+  duration: number;
+}
+
+export interface CacheOptions {
+  enabled: boolean;
+  ttl: number;
+  keyGenerator?: (job: Job) => string;
+}
+
+export interface QueueStats {
+  waiting: number;
+  active: number;
+  completed: number;
+  failed: number;
+  delayed: number;
+  paused: number;
+}
+
+export type JobHandler<T> = (job: Job<T>) => Promise<any>;
+export type BatchHandler<T> = (jobs: Job<T>[]) => Promise<any[]>;
+
+export interface BatchOptions {
+  batchSize: number;
+  maxWait?: number;
+}