@lobu/gateway 3.0.5 → 3.0.7

package/src/orchestration/message-consumer.ts

@@ -0,0 +1,584 @@
+import {
+  createChildSpan,
+  createLogger,
+  ErrorCode,
+  extractTraceId,
+  generateTraceId,
+  getTraceparent,
+  OrchestratorError,
+  retryWithBackoff,
+  SpanStatusCode,
+} from "@lobu/core";
+import * as Sentry from "@sentry/node";
+import type {
+  IMessageQueue,
+  QueueJob as SharedQueueJob,
+} from "../infrastructure/queue";
+import { RedisQueue, type RedisQueueConfig } from "../infrastructure/queue";
+import {
+  type BaseDeploymentManager,
+  buildCanonicalConversationKey,
+  generateDeploymentName,
+  type MessagePayload,
+  type OrchestratorConfig,
+} from "./base-deployment-manager";
+
+const logger = createLogger("orchestrator");
+
+export class MessageConsumer {
+  private queue: IMessageQueue;
+  private deploymentManager: BaseDeploymentManager;
+  private config: OrchestratorConfig;
+  private isRunning = false;
+  constructor(
+    config: OrchestratorConfig,
+    deploymentManager: BaseDeploymentManager
+  ) {
+    this.config = config;
+    this.deploymentManager = deploymentManager;
+    // Parse Redis connection string
+    const url = new URL(config.queues.connectionString);
+    if (url.protocol !== "redis:") {
+      throw new Error(
+        `Unsupported queue protocol: ${url.protocol}. Only redis:// is supported.`
+      );
+    }
+
+    const queueConfig: RedisQueueConfig = {
+      host: url.hostname,
+      port: Number.parseInt(url.port, 10) || 6379,
+      password: url.password || undefined,
+      db: url.pathname ? Number.parseInt(url.pathname.slice(1), 10) : 0,
+      maxRetriesPerRequest: 3,
+    };
+
+    this.queue = new RedisQueue(queueConfig);
+  }
+
+  async start(): Promise<void> {
+    try {
+      await this.queue.start();
+      this.isRunning = true;
+
+      // Create the messages queue if it doesn't exist
+      await this.queue.createQueue("messages");
+      logger.debug("Created/verified messages queue");
+
+      // Subscribe to the single messages queue for all messages
+      await this.queue.work(
+        "messages",
+        async (job: SharedQueueJob<MessagePayload>) => {
+          return await Sentry.startSpan(
+            {
+              name: "orchestrator.process_queue_job",
+              op: "orchestrator.queue_processing",
+              attributes: {
+                "job.id": job?.id || "unknown",
+              },
+            },
+            async () => {
+              return this.handleMessage(job);
+            }
+          );
+        }
+      );
+
+      logger.debug("Queue consumer started");
+    } catch (error) {
+      throw new OrchestratorError(
+        ErrorCode.QUEUE_JOB_PROCESSING_FAILED,
+        `Failed to start queue consumer: ${error instanceof Error ? error.message : String(error)}`,
+        { error },
+        true
+      );
+    }
+  }
+
+  async stop(): Promise<void> {
+    this.isRunning = false;
+    await this.queue.stop();
+  }
+
+  /**
+   * Handle all messages - creates deployment for new threads or routes to existing thread queues
+   */
+  private async handleMessage(
+    job: SharedQueueJob<MessagePayload>
+  ): Promise<void> {
+    const data = job?.data;
+    const jobId = job?.id || "unknown";
+
+    // Extract traceparent for distributed tracing (from message ingestion)
+    const traceparent = data?.platformMetadata?.traceparent as
+      | string
+      | undefined;
+
+    // Extract or generate trace ID for logging (backwards compatible)
+    const traceId =
+      extractTraceId(data) || generateTraceId(data?.messageId || jobId);
+
+    // Add traceId to Sentry scope for correlation
+    Sentry.getCurrentScope().setTag("traceId", traceId);
+
+    // Create child span for queue processing (linked to message_received span)
+    const queueSpan = createChildSpan("queue_processing", traceparent, {
+      "lobu.trace_id": traceId,
+      "lobu.job_id": jobId,
+      "lobu.user_id": data?.userId || "unknown",
+      "lobu.conversation_id": data?.conversationId || "unknown",
+    });
+
+    // Get traceparent to pass to worker (for further context propagation)
+    const childTraceparent = getTraceparent(queueSpan) || traceparent;
+
+    logger.info(
+      {
+        traceparent,
+        traceId,
+        jobId,
+        userId: data?.userId,
+        conversationId: data?.conversationId,
+      },
+      "Processing job with trace context"
+    );
+
+    try {
+      // CRITICAL: For consistent worker naming, conversationId must be the root conversation ID
+      // (e.g., Slack thread root ts), not individual message timestamps.
+      const effectiveConversationId = data.conversationId;
+      if (!effectiveConversationId) {
+        throw new OrchestratorError(
+          ErrorCode.QUEUE_JOB_PROCESSING_FAILED,
+          "conversationId is required for message routing",
+          { messageId: data.messageId, userId: data.userId },
+          true
+        );
+      }
+
+      const canonicalConversationKey = buildCanonicalConversationKey({
+        platform: data.platform,
+        channelId: data.channelId,
+        conversationId: effectiveConversationId,
+      });
+      const deploymentName = generateDeploymentName({
+        userId: data.userId,
+        platform: data.platform,
+        channelId: data.channelId,
+        conversationId: effectiveConversationId,
+      });
+
+      logger.info(
+        `Conversation routing - effectiveConversationId: ${effectiveConversationId}, canonicalKey: ${canonicalConversationKey}, deploymentName: ${deploymentName}`
+      );
+
+      // 1) Send to thread queue immediately (Redis persists; worker will drain on attach)
+      await Sentry.startSpan(
+        {
+          name: "orchestrator.send_to_worker_queue",
+          op: "orchestrator.message_routing",
+          attributes: {
+            "user.id": data.userId,
+            "conversation.id": effectiveConversationId || "unknown",
+            "deployment.name": deploymentName,
+          },
+        },
+        async () => {
+          await this.sendToWorkerQueue(data, deploymentName);
+        }
+      );
+
+      logger.info(
+        { traceId, traceparent: childTraceparent, deploymentName },
+        "Enqueued message to thread queue"
+      );
+
+      // 2) Ensure worker exists in the background (don't block queue send)
+      // Pass traceparent for propagation to worker deployment
+      this.ensureWorkerExists(
+        deploymentName,
+        data,
+        effectiveConversationId,
+        traceId,
+        childTraceparent
+      ).catch((bgError) => {
+        // Capture error for monitoring and alerting
+        Sentry.captureException(bgError, {
+          tags: {
+            component: "deployment-creation",
+            deploymentName,
+            userId: data.userId,
+            conversationId: effectiveConversationId,
+          },
+          level: "error",
+        });
+
+        logger.error(
+          {
+            traceId,
+            error: bgError instanceof Error ? bgError.message : String(bgError),
+            stack: bgError instanceof Error ? bgError.stack : undefined,
+            deploymentName,
+            userId: data.userId,
+            conversationId: effectiveConversationId,
+          },
+          "Critical: Background worker creation failed. Messages are queued but worker unavailable."
+        );
+
+        // Track failed deployments for monitoring and potential retry
+        this.trackFailedDeployment(deploymentName, data, bgError).catch(
+          (trackError) => {
+            logger.error("Failed to track deployment failure:", trackError);
+          }
+        );
+      });
+
+      queueSpan?.setStatus({ code: SpanStatusCode.OK });
+      queueSpan?.end();
+
+      logger.info({ traceId, jobId }, "Message job queued successfully");
+    } catch (error) {
+      queueSpan?.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: error instanceof Error ? error.message : String(error),
+      });
+      queueSpan?.end();
+      Sentry.captureException(error);
+      logger.error({ traceId, jobId, error }, "Message job failed");
+
+      // Re-throw for Redis retry handling
+      throw new OrchestratorError(
+        ErrorCode.QUEUE_JOB_PROCESSING_FAILED,
+        `Failed to process message job: ${error instanceof Error ? error.message : String(error)}`,
+        { jobId, data, error },
+        true
+      );
+    }
+  }
+
+  /**
+   * Send message to worker queue for the worker to consume
+   */
+  private async sendToWorkerQueue(
+    data: MessagePayload,
+    deploymentName: string
+  ): Promise<void> {
+    try {
+      // Create thread-specific queue name: thread_message_[deploymentid]
+      const threadQueueName = `thread_message_${deploymentName}`;
+
+      // Create the thread-specific queue if it doesn't exist
+      await this.queue.createQueue(threadQueueName);
+
+      // Send message to thread-specific queue
+      const jobId = await this.queue.send(threadQueueName, data, {
+        expireInSeconds: this.config.queues.expireInSeconds,
+        retryLimit: this.config.queues.retryLimit,
+        retryDelay: 2, // 2 seconds — fast retry for stale connection recovery
+        priority: 10, // Thread messages have high priority
+      });
+
+      if (!jobId) {
+        throw new OrchestratorError(
+          ErrorCode.QUEUE_JOB_PROCESSING_FAILED,
+          `queue.send() returned null/undefined for queue: ${threadQueueName}`,
+          { threadQueueName, deploymentName },
+          true
+        );
+      }
+
+      logger.info(
+        `✅ Sent message to thread queue ${threadQueueName} for conversation ${data.conversationId}, jobId: ${jobId}`
+      );
+    } catch (error) {
+      logger.error(`❌ [ERROR] sendToWorkerQueue failed:`, error);
+      throw new OrchestratorError(
+        ErrorCode.QUEUE_JOB_PROCESSING_FAILED,
+        `Failed to send message to thread queue: ${error instanceof Error ? error.message : String(error)}`,
+        { deploymentName, data, error },
+        true
+      );
+    }
+  }
+
+  /**
+   * Acquire a Redis-based lock for deployment creation.
+   * Prevents concurrent duplicate deployment creation for the same thread.
+   */
+  private async acquireDeploymentLock(
+    deploymentName: string
+  ): Promise<boolean> {
+    const lockKey = `deployment:lock:${deploymentName}`;
+    const redisClient = this.queue.getRedisClient();
+    // SET NX with 60s TTL - standard Redis distributed lock
+    const result = await redisClient.set(lockKey, "1", "EX", 60, "NX");
+    return result === "OK";
+  }
+
+  private async releaseDeploymentLock(deploymentName: string): Promise<void> {
+    const lockKey = `deployment:lock:${deploymentName}`;
+    const redisClient = this.queue.getRedisClient();
+    await redisClient.del(lockKey);
+  }
+
+  /**
+   * Ensure worker deployment exists for a thread
+   * Uses shared retry utility with linear backoff + jitter
+   * Uses Redis lock to prevent concurrent duplicate deployment creation
+   */
+  private async ensureWorkerExists(
+    deploymentName: string,
+    data: MessagePayload,
+    conversationId: string,
+    traceId: string,
+    traceparent?: string
+  ): Promise<void> {
+    return retryWithBackoff(
+      async () => {
+        // Ensure traceparent is in platformMetadata for worker deployment
+        const dataWithTrace: MessagePayload = {
+          ...data,
+          platformMetadata: {
+            ...data.platformMetadata,
+            traceparent: traceparent || data.platformMetadata?.traceparent,
+          },
+        };
+
+        // Check if this is truly a new thread by looking for existing deployment
+        const existingDeployments =
+          await this.deploymentManager.listDeployments();
+        const isNewThread = !existingDeployments.some(
+          (d) => d.deploymentName === deploymentName
+        );
+
+        if (isNewThread) {
+          // Acquire lock to prevent concurrent deployment creation
+          const acquired = await this.acquireDeploymentLock(deploymentName);
+          if (!acquired) {
+            logger.info(
+              { traceId, deploymentName },
+              "Another process is creating this deployment, waiting"
+            );
+            // Wait briefly and re-check - the other process should finish soon
+            await new Promise((r) => setTimeout(r, 3000));
+            // Verify it was created
+            const rechecked = await this.deploymentManager.listDeployments();
+            if (rechecked.some((d) => d.deploymentName === deploymentName)) {
+              await this.deploymentManager.scaleDeployment(deploymentName, 1);
+              logger.info(
+                { traceId, deploymentName },
+                "Deployment created by other process, scaled up"
+              );
+              await this.deploymentManager.updateDeploymentActivity(
+                deploymentName
+              );
+              return;
+            }
+            throw new Error("Deployment lock held but deployment not created");
+          }
+
+          try {
+            // Re-check after acquiring lock - another process may have created
+            // the deployment between our initial check and lock acquisition
+            const recheckAfterLock =
+              await this.deploymentManager.listDeployments();
+            if (
+              recheckAfterLock.some((d) => d.deploymentName === deploymentName)
+            ) {
+              logger.info(
+                { traceId, deploymentName },
+                "Deployment already created by another process after lock acquired"
+              );
+              await this.deploymentManager.scaleDeployment(deploymentName, 1);
+              await this.deploymentManager.updateDeploymentActivity(
+                deploymentName
+              );
+              return;
+            }
+
+            logger.info(
+              { traceId, traceparent, conversationId, deploymentName },
+              "New thread - creating deployment"
+            );
+            await this.deploymentManager.createWorkerDeployment(
+              data.userId,
+              conversationId,
+              dataWithTrace,
+              recheckAfterLock
+            );
+            logger.info({ traceId, deploymentName }, "Created deployment");
+          } finally {
+            try {
+              await this.releaseDeploymentLock(deploymentName);
+            } catch (releaseError) {
+              logger.error(
+                {
+                  deploymentName,
+                  error:
+                    releaseError instanceof Error
+                      ? releaseError.message
+                      : String(releaseError),
+                },
+                "CRITICAL: Failed to release deployment lock, attempting emergency Redis key deletion"
+              );
+              try {
+                const lockKey = `deployment:lock:${deploymentName}`;
+                const redisClient = this.queue.getRedisClient();
+                await redisClient.del(lockKey);
+                logger.info(
+                  { deploymentName },
+                  "Emergency lock cleanup succeeded"
+                );
+              } catch (emergencyError) {
+                logger.error(
+                  {
+                    deploymentName,
+                    error:
+                      emergencyError instanceof Error
+                        ? emergencyError.message
+                        : String(emergencyError),
+                  },
+                  "CRITICAL: Emergency lock cleanup also failed, lock will expire via TTL"
+                );
+              }
+            }
+          }
+        } else {
+          logger.info(
+            { traceId, conversationId, deploymentName },
+            "Existing thread - ensuring worker exists"
+          );
+          // Sync network config domains to grant store (picks up settings changes)
+          await this.deploymentManager.syncNetworkConfigGrants(dataWithTrace);
+          try {
+            await this.deploymentManager.scaleDeployment(deploymentName, 1);
+            logger.info(
+              { traceId, deploymentName },
+              "Scaled existing worker to 1"
+            );
+          } catch {
+            logger.info(
+              { traceId, conversationId, deploymentName },
+              "Worker doesn't exist, creating it"
+            );
+            await this.deploymentManager.createWorkerDeployment(
+              data.userId,
+              conversationId,
+              dataWithTrace
+            );
+            logger.info({ traceId, deploymentName }, "Created worker");
+          }
+        }
+
+        // Update deployment activity annotation for simplified tracking
+        await this.deploymentManager.updateDeploymentActivity(deploymentName);
+
+        logger.info({ traceId, deploymentName }, "Worker is ready");
+      },
+      {
+        maxRetries: 3,
+        baseDelay: 2000,
+        strategy: "linear",
+        jitter: true,
+        onRetry: (attempt, error) => {
+          logger.warn(
+            { traceId, deploymentName, attempt, maxAttempts: 3 },
+            `Retry attempt failed: ${error.message}`
+          );
+        },
+      }
+    );
+  }
+
+  /**
+   * Track failed deployment creation for monitoring and potential recovery.
+   * Also sends an error response to the user via the thread_response queue.
+   */
+  private async trackFailedDeployment(
+    deploymentName: string,
+    data: MessagePayload,
+    error: unknown
+  ): Promise<void> {
+    try {
+      const failureKey = `deployment:failed:${deploymentName}`;
+      const failureData = {
+        deploymentName,
+        userId: data.userId,
+        conversationId: data.conversationId,
+        error: error instanceof Error ? error.message : String(error),
+        stack: error instanceof Error ? error.stack : undefined,
+        timestamp: new Date().toISOString(),
+        queueName: `thread_message_${deploymentName}`,
+      };
+
+      // Store in Redis with 24h TTL for monitoring dashboards
+      // This allows ops to detect stuck queues and manually intervene
+      const redisClient = this.queue.getRedisClient();
+      await redisClient.setex(
+        failureKey,
+        86400, // 24 hours
+        JSON.stringify(failureData)
+      );
+
+      logger.info(
+        `Tracked deployment failure in Redis: ${failureKey} (TTL: 24h)`
+      );
+
+      const failureReason =
+        error instanceof Error ? error.message : String(error);
+      const isImagePullFailure =
+        /ImagePullBackOff|ErrImagePull|image pull/i.test(failureReason);
+      const userMessage = isImagePullFailure
+        ? "Worker startup failed due to a Kubernetes image pull error. Please retry after the deployment image/registry configuration is fixed."
+        : "Worker startup failed and your request could not be processed. Please retry in a moment.";
+
+      // Notify user that their message could not be processed
+      try {
+        const responseQueue = "thread_response";
+        await this.queue.createQueue(responseQueue);
+        await this.queue.send(responseQueue, {
+          messageId: data.messageId,
+          userId: data.userId,
+          channelId: data.channelId,
+          conversationId: data.conversationId,
+          platform: data.platform,
+          platformMetadata: data.platformMetadata,
+          content: userMessage,
+          processedMessageIds: [data.messageId],
+        });
+      } catch (notifyError) {
+        logger.error("Failed to send error notification to user:", notifyError);
+      }
+    } catch (trackError) {
+      // Don't fail the main flow if tracking fails
+      logger.error("Failed to track deployment failure:", trackError);
+    }
+  }
+
+  /**
+   * Get queue statistics
+   */
+  async getQueueStats(): Promise<{
+    messages?: {
+      waiting: number;
+      active: number;
+      completed: number;
+      failed: number;
+    };
+    isRunning: boolean;
+    error?: string;
+  }> {
+    try {
+      const stats = await this.queue.getQueueStats("messages");
+      return {
+        messages: stats,
+        isRunning: this.isRunning,
+      };
+    } catch (error) {
+      logger.error("Failed to get queue stats:", error);
+      return {
+        isRunning: this.isRunning,
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}