@lobu/gateway 3.0.5 → 3.0.6

package/src/infrastructure/queue/redis-queue.ts

@@ -0,0 +1,361 @@
+/**
+ * Redis-based message queue implementation using BullMQ
+ */
+
+import { createLogger } from "@lobu/core";
+import { type JobsOptions, Queue, type QueueEvents, Worker } from "bullmq";
+import Redis from "ioredis";
+import type {
+  IMessageQueue,
+  JobHandler,
+  QueueJob,
+  QueueOptions,
+  QueueStats,
+} from "./types";
+
+const logger = createLogger("redis-queue");
+
+export interface RedisQueueConfig {
+  host: string;
+  port: number;
+  password?: string;
+  db?: number;
+  maxRetriesPerRequest?: number;
+}
+
+export class RedisQueue implements IMessageQueue {
+  private config: RedisQueueConfig;
+  private queues: Map<string, Queue> = new Map();
+  private workers: Map<string, Worker> = new Map();
+  private pendingWorkers: Set<string> = new Set(); // Workers created with autorun:false, not yet started
+  private queueEvents: Map<string, QueueEvents> = new Map();
+  private isConnected = false;
+  private redisClient?: Redis;
+
+  constructor(config: RedisQueueConfig) {
+    this.config = config;
+  }
+
+  async start(): Promise<void> {
+    // Create shared Redis client for all queues and workers
+    this.redisClient = new Redis({
+      host: this.config.host,
+      port: this.config.port,
+      password: this.config.password,
+      db: this.config.db,
+      maxRetriesPerRequest: null, // Required by BullMQ
+    });
+
+    this.isConnected = true;
+    logger.debug("Redis queue started");
+  }
+
+  async stop(): Promise<void> {
+    this.isConnected = false;
+
+    // Close all workers first
+    for (const [name, worker] of this.workers.entries()) {
+      try {
+        worker.removeAllListeners();
+        await worker.close();
+        logger.debug(`Closed worker for queue: ${name}`);
+      } catch (error) {
+        logger.error(`Error closing worker ${name}:`, error);
+      }
+    }
+
+    // Close all queue events
+    for (const [name, events] of this.queueEvents.entries()) {
+      try {
+        await events.close();
+        logger.debug(`Closed events for queue: ${name}`);
+      } catch (error) {
+        logger.error(`Error closing events ${name}:`, error);
+      }
+    }
+
+    // Close all queues
+    for (const [name, queue] of this.queues.entries()) {
+      try {
+        await queue.close();
+        logger.debug(`Closed queue: ${name}`);
+      } catch (error) {
+        logger.error(`Error closing queue ${name}:`, error);
+      }
+    }
+
+    this.workers.clear();
+    this.queueEvents.clear();
+    this.queues.clear();
+
+    // Close the shared Redis client
+    if (this.redisClient) {
+      await this.redisClient.quit();
+      this.redisClient = undefined;
+    }
+
+    logger.debug("Redis queue stopped");
+  }
+
+  async createQueue(queueName: string): Promise<void> {
+    if (!this.queues.has(queueName)) {
+      if (!this.redisClient) {
+        throw new Error("Redis client not initialized. Call start() first.");
+      }
+
+      // Pass the shared Redis client to BullMQ
+      const queue = new Queue(queueName, {
+        connection: this.redisClient,
+      });
+
+      this.queues.set(queueName, queue);
+      logger.debug(`Created queue: ${queueName}`);
+    }
+  }
+
+  async send<T>(
+    queueName: string,
+    data: T,
+    options?: QueueOptions
+  ): Promise<string> {
+    if (!this.isConnected) {
+      throw new Error("Queue is not connected");
+    }
+
+    // Ensure queue exists
+    await this.createQueue(queueName);
+    const queue = this.queues.get(queueName);
+
+    if (!queue) {
+      throw new Error(`Queue ${queueName} not found`);
+    }
+
+    // Convert options to BullMQ format
+    const jobOptions: JobsOptions = {
+      priority: options?.priority,
+      attempts: options?.retryLimit ?? 3,
+      backoff: options?.retryDelay
+        ? {
+            type: "fixed",
+            delay: options.retryDelay * 1000, // Convert to milliseconds
+          }
+        : undefined,
+      removeOnComplete: {
+        age: 3600, // 1 hour
+        count: 1000,
+      },
+      removeOnFail: {
+        age: 24 * 3600, // 24 hours
+        count: 5000,
+      },
+      delay: options?.delayMs,
+    };
+
+    // Handle expiration
+    if (options?.expireInSeconds) {
+      jobOptions.removeOnComplete = {
+        age: options.expireInSeconds,
+        count: 1000,
+      };
+    }
+
+    // Handle singleton (deduplication)
+    if (options?.singletonKey) {
+      jobOptions.jobId = options.singletonKey;
+    }
+
+    const job = await queue.add(queueName, data, jobOptions);
+    logger.debug(`Enqueued job ${job.id} to queue ${queueName}`);
+
+    return job.id || "unknown";
+  }
+
+  async work<T>(
+    queueName: string,
+    handler: JobHandler<T>,
+    options?: { startPaused?: boolean }
+  ): Promise<void> {
+    if (!this.isConnected) {
+      throw new Error("Queue is not connected");
+    }
+
+    // Ensure queue exists
+    await this.createQueue(queueName);
+
+    // Close existing worker if one already exists for this queue
+    const existingWorker = this.workers.get(queueName);
+    if (existingWorker) {
+      existingWorker.removeAllListeners();
+      await existingWorker.close();
+      this.workers.delete(queueName);
+      this.pendingWorkers.delete(queueName);
+      logger.debug(`Closed existing worker for queue: ${queueName}`);
+    }
+
+    if (!this.redisClient) {
+      throw new Error("Redis client not initialized. Call start() first.");
+    }
+
+    const startPaused = options?.startPaused ?? false;
+
+    const worker = new Worker(
+      queueName,
+      async (job) => {
+        logger.debug(`Processing job ${job.id} from queue ${queueName}`);
+
+        const queueJob: QueueJob<T> = {
+          id: job.id || "unknown",
+          data: job.data as T,
+          name: job.name,
+        };
+
+        await handler(queueJob);
+      },
+      {
+        connection: this.redisClient,
+        concurrency: 1, // Process one job at a time per worker
+        ...(startPaused ? { autorun: false } : {}),
+      }
+    );
+
+    // Add event listeners for worker
+    worker.on("completed", (job) => {
+      logger.debug(`Job ${job.id} completed in queue ${queueName}`);
+    });
+
+    worker.on("failed", (job, err) => {
+      logger.error(`Job ${job?.id} failed in queue ${queueName}:`, err);
+
+      // Move to dead letter queue after all retries exhausted
+      if (job && job.attemptsMade >= (job.opts?.attempts ?? 3)) {
+        this.moveToDeadLetterQueue(queueName, job).catch((dlqErr) => {
+          logger.error(`Failed to move job ${job.id} to DLQ:`, dlqErr);
+        });
+      }
+    });
+
+    this.workers.set(queueName, worker);
+    if (startPaused) {
+      this.pendingWorkers.add(queueName);
+    }
+    logger.debug(
+      `Created worker for queue: ${queueName}${startPaused ? " (paused)" : ""}`
+    );
+  }
+
+  /**
+   * Move a failed job to the dead letter queue for later investigation/reprocessing
+   */
+  private async moveToDeadLetterQueue(
+    sourceQueue: string,
+    job: any
+  ): Promise<void> {
+    const dlqName = `${sourceQueue}:dlq`;
+    await this.createQueue(dlqName);
+    const dlq = this.queues.get(dlqName);
+    if (!dlq) return;
+
+    await dlq.add(
+      dlqName,
+      {
+        originalQueue: sourceQueue,
+        originalJobId: job.id,
+        data: job.data,
+        failedReason: job.failedReason,
+        attemptsMade: job.attemptsMade,
+        movedAt: new Date().toISOString(),
+      },
+      {
+        removeOnComplete: { age: 7 * 24 * 3600, count: 10000 }, // Keep DLQ jobs for 7 days
+        removeOnFail: false, // Never auto-remove DLQ failures
+      }
+    );
+
+    logger.info(
+      `Moved failed job ${job.id} from ${sourceQueue} to DLQ ${dlqName}`
+    );
+  }
+
+  async pauseWorker(queueName: string): Promise<void> {
+    const worker = this.workers.get(queueName);
+    if (!worker) {
+      logger.warn(
+        `Cannot pause worker for queue ${queueName} - worker not found`
+      );
+      return;
+    }
+
+    if (worker.isPaused()) {
+      logger.debug(`Worker for queue ${queueName} is already paused`);
+      return;
+    }
+
+    await worker.pause();
+    logger.debug(`Paused worker for queue ${queueName}`);
+  }
+
+  async resumeWorker(queueName: string): Promise<void> {
+    const worker = this.workers.get(queueName);
+    if (!worker) {
+      logger.warn(
+        `Cannot resume worker for queue ${queueName} - worker not found`
+      );
+      return;
+    }
+
+    // First-time start: worker was created with autorun:false and never started
+    if (this.pendingWorkers.has(queueName)) {
+      this.pendingWorkers.delete(queueName);
+      worker.run();
+      logger.debug(`Started worker for queue ${queueName} (first run)`);
+      return;
+    }
+
+    if (!worker.isPaused()) {
+      logger.debug(`Worker for queue ${queueName} is already running`);
+      return;
+    }
+
+    worker.resume();
+    logger.debug(`Resumed worker for queue ${queueName}`);
+  }
+
+  /**
+   * Get detailed queue statistics
+   */
+  async getQueueStats(queueName: string): Promise<QueueStats> {
+    const queue = this.queues.get(queueName);
+    if (!queue) {
+      return { waiting: 0, active: 0, completed: 0, failed: 0 };
+    }
+
+    const counts = await queue.getJobCounts(
+      "waiting",
+      "active",
+      "completed",
+      "failed"
+    );
+
+    return {
+      waiting: counts.waiting ?? 0,
+      active: counts.active ?? 0,
+      completed: counts.completed ?? 0,
+      failed: counts.failed ?? 0,
+    };
+  }
+
+  isHealthy(): boolean {
+    return this.isConnected;
+  }
+
+  /**
+   * Get underlying Redis client for general-purpose Redis operations
+   * Returns the shared IORedis client used by all queues and workers
+   */
+  getRedisClient(): any {
+    if (!this.redisClient) {
+      throw new Error("Redis client not initialized. Call start() first.");
+    }
+    return this.redisClient;
+  }
+}

package/src/infrastructure/queue/types.ts

@@ -0,0 +1,133 @@
+/**
+ * Message queue interface and payload types for lobu
+ * Supports multiple queue backends (currently Redis via BullMQ)
+ */
+
+// ============================================================================
+// Queue Types
+// ============================================================================
+
+export interface QueueJob<T = any> {
+  id: string;
+  data: T;
+  name?: string;
+}
+
+export interface QueueOptions {
+  priority?: number;
+  retryLimit?: number;
+  retryDelay?: number;
+  expireInSeconds?: number;
+  singletonKey?: string;
+  /** Delay in milliseconds before the job is processed */
+  delayMs?: number;
+}
+
+export interface QueueStats {
+  waiting: number;
+  active: number;
+  completed: number;
+  failed: number;
+}
+
+export type JobHandler<T = any> = (job: QueueJob<T>) => Promise<void>;
+
+/**
+ * Abstract message queue interface
+ * Implementations: RedisQueue (BullMQ)
+ */
+export interface IMessageQueue {
+  /**
+   * Start the queue (connect to backend)
+   */
+  start(): Promise<void>;
+
+  /**
+   * Stop the queue (disconnect from backend)
+   */
+  stop(): Promise<void>;
+
+  /**
+   * Create a queue if it doesn't exist
+   */
+  createQueue(queueName: string): Promise<void>;
+
+  /**
+   * Send a message to a queue
+   */
+  send<T>(queueName: string, data: T, options?: QueueOptions): Promise<string>;
+
+  /**
+   * Subscribe to a queue and process jobs.
+   * @param startPaused - If true, worker is created but won't process jobs until resumeWorker() is called.
+   */
+  work<T>(
+    queueName: string,
+    handler: JobHandler<T>,
+    options?: { startPaused?: boolean }
+  ): Promise<void>;
+
+  /**
+   * Pause a queue worker (stops processing jobs)
+   */
+  pauseWorker(queueName: string): Promise<void>;
+
+  /**
+   * Resume a queue worker (starts processing jobs)
+   */
+  resumeWorker(queueName: string): Promise<void>;
+
+  /**
+   * Get detailed queue statistics
+   */
+  getQueueStats(queueName: string): Promise<QueueStats>;
+
+  /**
+   * Check if queue is healthy/connected
+   */
+  isHealthy(): boolean;
+
+  /**
+   * Get underlying Redis client for general-purpose Redis operations
+   * Used for application state storage (sessions, cache, etc.)
+   */
+  getRedisClient(): any;
+}
+
+// ============================================================================
+// Payload Types
+// ============================================================================
+
+/**
+ * Shared payload contract for worker → platform thread responses.
+ * Ensures gateway consumers and workers stay type-aligned.
+ */
+export interface ThreadResponsePayload {
+  messageId: string;
+  channelId: string;
+  conversationId: string;
+  userId: string;
+  teamId: string;
+  platform?: string; // Platform identifier (slack, whatsapp, api, etc.) for multi-platform routing
+  content?: string; // Used only for ephemeral messages (OAuth/auth flows)
+  delta?: string;
+  isFullReplacement?: boolean;
+  processedMessageIds?: string[];
+  error?: string;
+  errorCode?: string;
+  timestamp: number;
+  originalMessageId?: string;
+  moduleData?: Record<string, unknown>;
+  botResponseId?: string;
+  ephemeral?: boolean;
+  statusUpdate?: {
+    elapsedSeconds: number;
+    state: string;
+  };
+  platformMetadata?: Record<string, unknown>; // Platform-specific metadata (e.g., sessionId for API)
+
+  // Exec-specific response fields (for jobType === "exec")
+  execId?: string; // Exec job ID for response routing
+  execStream?: "stdout" | "stderr"; // Which stream this delta is from
+  execExitCode?: number; // Process exit code (sent on completion)
+}

package/src/infrastructure/redis/system-message-limiter.ts

@@ -0,0 +1,94 @@
+import { createLogger } from "@lobu/core";
+import type Redis from "ioredis";
+
+const logger = createLogger("system-message-limiter");
+
+export interface SendOnceOptions {
+  /**
+   * How long to remember that this message was successfully sent.
+   * This is the main debounce window.
+   */
+  sentTtlSeconds: number;
+  /**
+   * How long to hold an in-flight lock to prevent concurrent duplicate sends.
+   * Keep short (10-60s) to bound crash edge cases.
+   */
+  lockTtlSeconds?: number;
+  /**
+   * If Redis is unavailable, allow the send to proceed (fail-open) or suppress it (fail-closed).
+   * For spammy system prompts, fail-closed is usually safer.
+   */
+  failOpen?: boolean;
+}
+
+/**
+ * Generic idempotency gate for outbound "system" messages (auth/setup prompts, warnings, etc.).
+ *
+ * Pattern:
+ * - `sentKey` is written only after a successful send.
+ * - `lockKey` prevents concurrent callers from sending duplicates while the first send is in-flight.
+ *
+ * This is intentionally small and Redis-only (no in-memory fallback) so behavior is consistent
+ * across replicas.
+ */
+export class SystemMessageLimiter {
+  constructor(
+    private redis: Redis,
+    private namespace: string = "sysmsg"
+  ) {}
+
+  async sendOnce(
+    dedupeKey: string,
+    sendFn: () => Promise<void>,
+    options: SendOnceOptions
+  ): Promise<boolean> {
+    const sentTtlSeconds = Math.max(1, options.sentTtlSeconds);
+    const lockTtlSeconds = Math.max(1, options.lockTtlSeconds ?? 30);
+    const failOpen = options.failOpen ?? false;
+
+    const sentKey = `${this.namespace}:sent:${dedupeKey}`;
+    const lockKey = `${this.namespace}:lock:${dedupeKey}`;
+
+    try {
+      const alreadySent = (await this.redis.exists(sentKey)) === 1;
+      if (alreadySent) {
+        return false;
+      }
+
+      // Use a lock so concurrent callers (or retries) don't double-send before sentKey is written.
+      const lockResult = await this.redis.set(
+        lockKey,
+        "1",
+        "EX",
+        lockTtlSeconds,
+        "NX"
+      );
+      if (lockResult !== "OK") {
+        return false;
+      }
+    } catch (error) {
+      logger.warn(
+        { error: String(error), sentKey, lockKey, failOpen },
+        "Redis unavailable while gating system message"
+      );
+      return failOpen;
+    }
+
+    try {
+      await sendFn();
+      // Mark as sent only after successful delivery attempt.
+      await this.redis.set(sentKey, "1", "EX", sentTtlSeconds);
+      return true;
+    } finally {
+      // Best-effort lock cleanup; key also expires automatically.
+      try {
+        await this.redis.del(lockKey);
+      } catch (error) {
+        logger.debug(
+          { error: String(error), lockKey },
+          "Failed to release system-message lock"
+        );
+      }
+    }
+  }
+}