flowfn 0.0.1

package/src/stream/stream.ts

@@ -0,0 +1,194 @@
+import {
+  Stream,
+  Message,
+  PublishOptions,
+  SubscribeOptions,
+  MessageHandler,
+  Subscription,
+  ConsumerOptions,
+  Consumer,
+  StreamInfo,
+  TrimStrategy,
+  StreamOptions,
+} from "./types.js";
+import { FlowAdapter } from "../adapters/base.js";
+import { v4 as uuidv4 } from "uuid";
+
+export class StreamImpl<T = any> implements Stream<T> {
+  name: string;
+  private adapter: FlowAdapter;
+  private options: StreamOptions;
+  private messages: Map<string, Message<T>> = new Map();
+
+  constructor(name: string, adapter: FlowAdapter, options: StreamOptions = {}) {
+    this.name = name;
+    this.adapter = adapter;
+    this.options = options;
+  }
+
+  async publish(data: T, options?: PublishOptions): Promise<string> {
+    const message: Message<T> = {
+      id: uuidv4(),
+      stream: this.name,
+      data,
+      headers: options?.headers,
+      timestamp: Date.now(),
+      partition: options?.partition,
+      key: options?.key,
+      ack: async () => {},
+      nack: async () => {},
+    };
+
+    // Store message locally for retrieval
+    this.messages.set(message.id, message);
+
+    // Auto-trim if maxLength is set
+    if (this.options.maxLength && this.messages.size > this.options.maxLength) {
+      await this.trim({ maxLength: this.options.maxLength });
+    }
+
+    return this.adapter.publish(this.name, message);
+  }
+
+  async publishBatch(
+    messages: Array<{ data: T; options?: PublishOptions }>
+  ): Promise<string[]> {
+    return Promise.all(messages.map((m) => this.publish(m.data, m.options)));
+  }
+
+  async subscribe(
+    handler: MessageHandler<T>,
+    options?: SubscribeOptions
+  ): Promise<Subscription> {
+    return this.adapter.subscribe(this.name, handler as any);
+  }
+
+  createConsumer(consumerId: string, options: ConsumerOptions): Consumer<T> {
+    let subscription: Subscription | null = null;
+    let paused = false;
+
+    return {
+      subscribe: async (handler: MessageHandler<T>) => {
+        // If fromBeginning, replay existing messages first
+        if (options.fromBeginning && this.messages.size > 0) {
+          const sortedMessages = Array.from(this.messages.values()).sort(
+            (a, b) => a.timestamp - b.timestamp
+          );
+
+          for (const msg of sortedMessages) {
+            if (!paused) {
+              await handler(msg).catch(console.error);
+            }
+          }
+        }
+
+        subscription = await this.adapter.consume(
+          this.name,
+          options.groupId,
+          consumerId,
+          handler as any
+        );
+      },
+      pause: async () => {
+        paused = true;
+      },
+      resume: async () => {
+        paused = false;
+      },
+      close: async () => {
+        if (subscription) await subscription.unsubscribe();
+      },
+    };
+  }
+
+  async getInfo(): Promise<StreamInfo> {
+    const info = await this.adapter.getStreamInfo(this.name);
+    return {
+      ...info,
+      length: this.messages.size,
+    };
+  }
+
+  async trim(strategy: TrimStrategy): Promise<number> {
+    const now = Date.now();
+    const messagesToDelete: string[] = [];
+
+    if (strategy.maxLength) {
+      // Sort by timestamp and keep only the newest maxLength messages
+      const sortedMessages = Array.from(this.messages.entries()).sort(
+        (a, b) => b[1].timestamp - a[1].timestamp
+      );
+
+      if (sortedMessages.length > strategy.maxLength) {
+        const toRemove = sortedMessages.slice(strategy.maxLength);
+        messagesToDelete.push(...toRemove.map(([id]) => id));
+      }
+    }
+
+    if (strategy.maxAgeSeconds) {
+      const maxAge = strategy.maxAgeSeconds * 1000;
+      for (const [id, message] of this.messages.entries()) {
+        if (now - message.timestamp > maxAge) {
+          messagesToDelete.push(id);
+        }
+      }
+    }
+
+    // Remove duplicates
+    const uniqueToDelete = [...new Set(messagesToDelete)];
+    for (const id of uniqueToDelete) {
+      this.messages.delete(id);
+    }
+
+    return uniqueToDelete.length;
+  }
+
+  async getMessages(
+    start: string,
+    end: string,
+    count?: number
+  ): Promise<Message<T>[]> {
+    const allMessages = Array.from(this.messages.values()).sort(
+      (a, b) => a.timestamp - b.timestamp
+    );
+
+    // Filter by ID range (lexicographic comparison)
+    let filtered = allMessages.filter((m) => m.id >= start && m.id <= end);
+
+    // Apply count limit if specified
+    if (count !== undefined && count > 0) {
+      filtered = filtered.slice(0, count);
+    }
+
+    return filtered;
+  }
+
+  /**
+   * Replay messages from a specific timestamp
+   */
+  async replay(
+    fromTimestamp: number,
+    handler: MessageHandler<T>
+  ): Promise<number> {
+    const messages = Array.from(this.messages.values())
+      .filter((m) => m.timestamp >= fromTimestamp)
+      .sort((a, b) => a.timestamp - b.timestamp);
+
+    for (const message of messages) {
+      await handler(message);
+    }
+
+    return messages.length;
+  }
+
+  /**
+   * Get message count
+   */
+  getMessageCount(): number {
+    return this.messages.size;
+  }
+
+  async close(): Promise<void> {
+    this.messages.clear();
+  }
+}

package/src/stream/types.ts

@@ -0,0 +1,81 @@
+export interface StreamOptions {
+    maxLength?: number;
+    retention?: number;
+    partitions?: number;
+}
+
+export interface Stream<T = any> {
+  name: string;
+  
+  // Publishing
+  publish(data: T, options?: PublishOptions): Promise<string>;
+  publishBatch(messages: Array<{ data: T; options?: PublishOptions }>): Promise<string[]>;
+  
+  // Subscribing
+  subscribe(handler: MessageHandler<T>, options?: SubscribeOptions): Promise<Subscription>;
+  createConsumer(consumerId: string, options: ConsumerOptions): Consumer<T>;
+  
+  // Management
+  getInfo(): Promise<StreamInfo>;
+  trim(strategy: TrimStrategy): Promise<number>;
+  getMessages(start: string, end: string, count?: number): Promise<Message<T>[]>;
+  
+  // Lifecycle
+  close(): Promise<void>;
+}
+
+export interface Message<T = any> {
+  id: string;
+  stream: string;
+  data: T;
+  headers?: Record<string, string>;
+  timestamp: number;
+  partition?: number;
+  offset?: number;
+  key?: string;
+  
+  ack(): Promise<void>;
+  nack(requeue?: boolean): Promise<void>;
+}
+
+export interface PublishOptions {
+  key?: string;
+  headers?: Record<string, string>;
+  partition?: number;
+}
+
+export interface SubscribeOptions {
+    // Basic subscription options if any
+}
+
+export interface ConsumerOptions {
+  groupId: string;
+  fromBeginning?: boolean;
+  autoCommit?: boolean;
+  commitInterval?: number;
+  maxInFlight?: number;
+}
+
+export interface Consumer<T = any> {
+  subscribe(handler: MessageHandler<T>): Promise<void>;
+  pause(): Promise<void>;
+  resume(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export type MessageHandler<T> = (message: Message<T>) => Promise<void>;
+
+export interface Subscription {
+    unsubscribe(): Promise<void>;
+}
+
+export interface StreamInfo {
+    name: string;
+    length: number;
+    groups: number;
+}
+
+export interface TrimStrategy {
+    maxLength?: number;
+    maxAgeSeconds?: number;
+}

package/src/utils/hashing.ts

@@ -0,0 +1,29 @@
+import { createHash } from "crypto";
+
+/**
+ * Generate a hash for job deduplication
+ */
+export function hashJob(data: any, options?: { algorithm?: string }): string {
+  const algorithm = options?.algorithm || "sha256";
+  const hash = createHash(algorithm);
+
+  // Normalize and stringify the data for consistent hashing
+  const normalized = JSON.stringify(data, Object.keys(data).sort());
+  hash.update(normalized);
+
+  return hash.digest("hex");
+}
+
+/**
+ * Generate a deduplication key from job name and data
+ */
+export function generateDeduplicationKey(name: string, data: any): string {
+  return `${name}:${hashJob(data)}`;
+}
+
+/**
+ * Check if two job payloads are equivalent
+ */
+export function areJobsEquivalent(job1: any, job2: any): boolean {
+  return hashJob(job1) === hashJob(job2);
+}

package/src/utils/id-generator.ts

@@ -0,0 +1,109 @@
+import { v4 as uuidv4, v5 as uuidv5, v1 as uuidv1 } from "uuid";
+import { createHash, randomBytes } from "crypto";
+
+/**
+ * ID generation strategies
+ */
+export type IdStrategy =
+  | "uuid-v4"
+  | "uuid-v5"
+  | "uuid-v1"
+  | "nanoid"
+  | "incremental"
+  | "custom";
+
+export interface IdGeneratorOptions {
+  strategy?: IdStrategy;
+  namespace?: string; // For UUID v5
+  prefix?: string;
+  customGenerator?: () => string;
+}
+
+/**
+ * Generate a unique identifier based on the specified strategy
+ */
+export function generateId(options: IdGeneratorOptions = {}): string {
+  const {
+    strategy = "uuid-v4",
+    namespace,
+    prefix = "",
+    customGenerator,
+  } = options;
+
+  let id: string;
+
+  switch (strategy) {
+    case "uuid-v4":
+      id = uuidv4();
+      break;
+
+    case "uuid-v5":
+      if (!namespace) {
+        throw new Error("UUID v5 requires a namespace");
+      }
+      const name = `${Date.now()}-${randomBytes(8).toString("hex")}`;
+      id = uuidv5(name, namespace);
+      break;
+
+    case "uuid-v1":
+      id = uuidv1();
+      break;
+
+    case "nanoid":
+      // Simple nanoid-like implementation
+      id = randomBytes(16).toString("base64url");
+      break;
+
+    case "incremental":
+      // Simple timestamp-based incremental ID
+      id = `${Date.now()}-${randomBytes(4).toString("hex")}`;
+      break;
+
+    case "custom":
+      if (!customGenerator) {
+        throw new Error("Custom strategy requires customGenerator function");
+      }
+      id = customGenerator();
+      break;
+
+    default:
+      id = uuidv4();
+  }
+
+  return prefix ? `${prefix}${id}` : id;
+}
+
+/**
+ * Generate a job ID with optional prefix
+ */
+export function generateJobId(prefix?: string): string {
+  return generateId({ prefix: prefix || "job_" });
+}
+
+/**
+ * Generate a workflow execution ID
+ */
+export function generateExecutionId(prefix?: string): string {
+  return generateId({ prefix: prefix || "exec_" });
+}
+
+/**
+ * Generate a message ID for streams
+ */
+export function generateMessageId(prefix?: string): string {
+  return generateId({ prefix: prefix || "msg_" });
+}
+
+/**
+ * Generate a deterministic ID based on content (useful for idempotency)
+ */
+export function generateDeterministicId(
+  content: string | object,
+  prefix?: string
+): string {
+  const hash = createHash("sha256");
+  const data = typeof content === "string" ? content : JSON.stringify(content);
+  hash.update(data);
+  const id = hash.digest("hex").slice(0, 32);
+  return prefix ? `${prefix}${id}` : id;
+}

package/src/utils/serialization.ts

@@ -0,0 +1,142 @@
+/**
+ * Serialization utilities for FlowFn
+ */
+
+export interface SerializationOptions {
+  pretty?: boolean;
+  includeUndefined?: boolean;
+  dateFormat?: "iso" | "timestamp";
+}
+
+/**
+ * Serialize job data to JSON string
+ */
+export function serialize<T = any>(
+  data: T,
+  options: SerializationOptions = {}
+): string {
+  const { pretty = false, dateFormat = "iso" } = options;
+
+  const replacer = (key: string, value: any) => {
+    // Handle dates
+    if (value instanceof Date) {
+      return dateFormat === "timestamp" ? value.getTime() : value.toISOString();
+    }
+
+    // Handle undefined
+    if (value === undefined && !options.includeUndefined) {
+      return null;
+    }
+
+    // Handle functions (skip them)
+    if (typeof value === "function") {
+      return undefined;
+    }
+
+    // Handle BigInt
+    if (typeof value === "bigint") {
+      return value.toString();
+    }
+
+    return value;
+  };
+
+  return JSON.stringify(data, replacer, pretty ? 2 : undefined);
+}
+
+/**
+ * Deserialize JSON string to object
+ */
+export function deserialize<T = any>(json: string): T {
+  return JSON.parse(json, (key, value) => {
+    // Try to parse ISO date strings
+    if (typeof value === "string") {
+      const isoDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?$/;
+      if (isoDateRegex.test(value)) {
+        return new Date(value);
+      }
+    }
+    return value;
+  });
+}
+
+/**
+ * Safely serialize data with circular reference handling
+ */
+export function serializeSafe<T = any>(
+  data: T,
+  options: SerializationOptions = {}
+): string {
+  const seen = new WeakSet();
+  const { pretty = false } = options;
+
+  const replacer = (key: string, value: any) => {
+    if (typeof value === "object" && value !== null) {
+      if (seen.has(value)) {
+        return "[Circular]";
+      }
+      seen.add(value);
+    }
+
+    // Handle dates
+    if (value instanceof Date) {
+      return value.toISOString();
+    }
+
+    // Handle functions
+    if (typeof value === "function") {
+      return "[Function]";
+    }
+
+    // Handle BigInt
+    if (typeof value === "bigint") {
+      return value.toString();
+    }
+
+    return value;
+  };
+
+  return JSON.stringify(data, replacer, pretty ? 2 : undefined);
+}
+
+/**
+ * Clone an object using serialization
+ */
+export function cloneViaSerialization<T>(obj: T): T {
+  return deserialize(serialize(obj));
+}
+
+/**
+ * Serialize with compression (base64 encode)
+ */
+export function serializeCompressed<T = any>(data: T): string {
+  const json = serialize(data);
+  return Buffer.from(json).toString("base64");
+}
+
+/**
+ * Deserialize compressed data
+ */
+export function deserializeCompressed<T = any>(compressed: string): T {
+  const json = Buffer.from(compressed, "base64").toString("utf-8");
+  return deserialize(json);
+}
+
+/**
+ * Check if a value is serializable
+ */
+export function isSerializable(value: any): boolean {
+  try {
+    serialize(value);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Get size of serialized data in bytes
+ */
+export function getSerializedSize(data: any): number {
+  return Buffer.byteLength(serialize(data), "utf-8");
+}

package/src/utils/time.ts

@@ -0,0 +1,167 @@
+/**
+ * Time utility functions for FlowFn
+ */
+
+/**
+ * Duration units
+ */
+export interface Duration {
+  milliseconds?: number;
+  seconds?: number;
+  minutes?: number;
+  hours?: number;
+  days?: number;
+  weeks?: number;
+}
+
+/**
+ * Convert duration object to milliseconds
+ */
+export function toMilliseconds(duration: Duration): number {
+  let ms = 0;
+
+  if (duration.milliseconds) ms += duration.milliseconds;
+  if (duration.seconds) ms += duration.seconds * 1000;
+  if (duration.minutes) ms += duration.minutes * 60 * 1000;
+  if (duration.hours) ms += duration.hours * 60 * 60 * 1000;
+  if (duration.days) ms += duration.days * 24 * 60 * 60 * 1000;
+  if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1000;
+
+  return ms;
+}
+
+/**
+ * Convert milliseconds to duration object
+ */
+export function fromMilliseconds(ms: number): Duration {
+  const weeks = Math.floor(ms / (7 * 24 * 60 * 60 * 1000));
+  ms %= 7 * 24 * 60 * 60 * 1000;
+
+  const days = Math.floor(ms / (24 * 60 * 60 * 1000));
+  ms %= 24 * 60 * 60 * 1000;
+
+  const hours = Math.floor(ms / (60 * 60 * 1000));
+  ms %= 60 * 60 * 1000;
+
+  const minutes = Math.floor(ms / (60 * 1000));
+  ms %= 60 * 1000;
+
+  const seconds = Math.floor(ms / 1000);
+  const milliseconds = ms % 1000;
+
+  return { weeks, days, hours, minutes, seconds, milliseconds };
+}
+
+/**
+ * Format duration as human-readable string
+ */
+export function formatDuration(duration: Duration): string {
+  const parts: string[] = [];
+
+  if (duration.weeks) parts.push(`${duration.weeks}w`);
+  if (duration.days) parts.push(`${duration.days}d`);
+  if (duration.hours) parts.push(`${duration.hours}h`);
+  if (duration.minutes) parts.push(`${duration.minutes}m`);
+  if (duration.seconds) parts.push(`${duration.seconds}s`);
+  if (duration.milliseconds) parts.push(`${duration.milliseconds}ms`);
+
+  return parts.join(" ") || "0ms";
+}
+
+/**
+ * Sleep for specified milliseconds
+ */
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Sleep for specified duration
+ */
+export async function sleepDuration(duration: Duration): Promise<void> {
+  return sleep(toMilliseconds(duration));
+}
+
+/**
+ * Create a timeout promise
+ */
+export function timeout<T>(
+  promise: Promise<T>,
+  ms: number,
+  message?: string
+): Promise<T> {
+  return Promise.race([
+    promise,
+    new Promise<T>((_, reject) =>
+      setTimeout(
+        () => reject(new Error(message || `Timeout after ${ms}ms`)),
+        ms
+      )
+    ),
+  ]);
+}
+
+/**
+ * Get current timestamp in milliseconds
+ */
+export function now(): number {
+  return Date.now();
+}
+
+/**
+ * Calculate delay until a specific timestamp
+ */
+export function delayUntil(timestamp: number): number {
+  return Math.max(0, timestamp - Date.now());
+}
+
+/**
+ * Check if a timestamp is in the past
+ */
+export function isPast(timestamp: number): boolean {
+  return timestamp < Date.now();
+}
+
+/**
+ * Check if a timestamp is in the future
+ */
+export function isFuture(timestamp: number): boolean {
+  return timestamp > Date.now();
+}
+
+/**
+ * Add duration to timestamp
+ */
+export function addDuration(timestamp: number, duration: Duration): number {
+  return timestamp + toMilliseconds(duration);
+}
+
+/**
+ * Parse cron-like duration string (e.g., "5m", "1h", "30s")
+ */
+export function parseDuration(str: string): number {
+  const match = str.match(/^(\d+)(ms|s|m|h|d|w)$/);
+  if (!match) {
+    throw new Error(`Invalid duration string: ${str}`);
+  }
+
+  const value = parseInt(match[1], 10);
+  const unit = match[2];
+
+  switch (unit) {
+    case "ms":
+      return value;
+    case "s":
+      return value * 1000;
+    case "m":
+      return value * 60 * 1000;
+    case "h":
+      return value * 60 * 60 * 1000;
+    case "d":
+      return value * 24 * 60 * 60 * 1000;
+    case "w":
+      return value * 7 * 24 * 60 * 60 * 1000;
+    default:
+      throw new Error(`Unknown unit: ${unit}`);
+  }
+}