@ductape/sdk 0.0.4-v99 → 0.0.5

package/dist/brokers/brokers.service.js

@@ -22,6 +22,8 @@ const processorApi_service_1 = require("../api/services/processorApi.service");
 const secrets_1 = require("../secrets");
 const cache_manager_1 = require("../cache/cache.manager");
 const sessions_1 = require("../sessions");
+/** Log prefix for broker operations */
+const LOG_PREFIX = '[BrokersService]';
 /**
  * Error class for broker operations
  */
@@ -72,6 +74,7 @@ exports.BrokerError = BrokerError;
  */
 class BrokersService {
     constructor(config) {
+        var _a;
         this.logService = null;
         this.productId = '';
         this.productTag = '';
@@ -80,14 +83,195 @@ class BrokersService {
         this.cacheManager = null;
         /** Private keys per product for cache encryption */
         this.privateKeys = new Map();
+        /** Connection pool for broker services - keyed by broker type + config hash */
+        this.brokerServicePool = new Map();
+        /** Cache for initialized broker configs - keyed by "productTag:brokerTag:envSlug" */
+        this.initializedBrokerCache = new Map();
+        /** Cache for initialized products - keyed by productTag */
+        this.initializedProducts = new Set();
+        /** Cache for registered broker consumers - keyed by consumerTag to track which consumers have been registered */
+        this.registeredConsumers = new Set();
+        /** Local cache for cache configurations to avoid repeated API calls */
+        this.cacheConfigCache = new Map();
+        /** Dedupe "Publish to broker - success" when same event is published twice in quick succession (e.g. workflow step run twice) */
+        this.lastPublishSuccessLogKey = null;
+        this.lastPublishSuccessLogTime = 0;
+        // ============================================================================
+        // BROKER MESSAGE QUERY METHODS (for workbench with decryption)
+        // ============================================================================
+        /**
+         * Messages namespace for workbench broker message queries
+         * All methods decrypt message content before returning
+         */
+        this.messages = {
+            /**
+             * Query broker messages with filtering and pagination
+             * Decrypts message content before returning
+             */
+            query: async (options) => {
+                try {
+                    await this.initializeProductForEvents(options.product);
+                    const result = await this.processorApiService.queryBrokerMessages({
+                        product_tag: options.product,
+                        env: options.env,
+                        broker_tag: options.brokerTag,
+                        topic_tag: options.topicTag,
+                        producer_tag: options.producerTag,
+                        consumer_tag: options.consumerTag,
+                        status: options.status,
+                        start_date: options.startDate,
+                        end_date: options.endDate,
+                        page: options.page,
+                        limit: options.limit,
+                    }, this.getUserAccess());
+                    // Decrypt message content for each message
+                    const messages = result.messages.map((msg) => (Object.assign(Object.assign({}, msg), { message_decrypted: msg.message_encrypted
+                            ? this.decryptPayload(msg.message_encrypted)
+                            : undefined })));
+                    return {
+                        messages,
+                        total: result.total,
+                        page: result.page,
+                        limit: result.limit,
+                        hasMore: result.hasMore,
+                    };
+                }
+                catch (error) {
+                    if (error instanceof BrokerError)
+                        throw error;
+                    throw new BrokerError(`Failed to query broker messages: ${error}`, 'QUERY_MESSAGES_FAILED', error);
+                }
+            },
+            /**
+             * Get broker producers with pagination
+             */
+            getProducers: async (options) => {
+                try {
+                    await this.initializeProductForEvents(options.product);
+                    return await this.processorApiService.getBrokerProducers({
+                        product_tag: options.product,
+                        env: options.env,
+                        broker_tag: options.brokerTag,
+                        topic_tag: options.topicTag,
+                        page: options.page,
+                        limit: options.limit,
+                    }, this.getUserAccess());
+                }
+                catch (error) {
+                    if (error instanceof BrokerError)
+                        throw error;
+                    throw new BrokerError(`Failed to get broker producers: ${error}`, 'GET_PRODUCERS_FAILED', error);
+                }
+            },
+            /**
+             * Get broker consumers with pagination
+             */
+            getConsumers: async (options) => {
+                try {
+                    await this.initializeProductForEvents(options.product);
+                    return await this.processorApiService.getBrokerConsumers({
+                        product_tag: options.product,
+                        env: options.env,
+                        broker_tag: options.brokerTag,
+                        topic_tag: options.topicTag,
+                        page: options.page,
+                        limit: options.limit,
+                    }, this.getUserAccess());
+                }
+                catch (error) {
+                    if (error instanceof BrokerError)
+                        throw error;
+                    throw new BrokerError(`Failed to get broker consumers: ${error}`, 'GET_CONSUMERS_FAILED', error);
+                }
+            },
+            /**
+             * Get broker dead letters with pagination
+             * Decrypts message content before returning
+             */
+            getDeadLetters: async (options) => {
+                try {
+                    await this.initializeProductForEvents(options.product);
+                    const result = await this.processorApiService.getBrokerDeadLetters({
+                        product_tag: options.product,
+                        env: options.env,
+                        broker_tag: options.brokerTag,
+                        topic_tag: options.topicTag,
+                        consumer_tag: options.consumerTag,
+                        start_date: options.startDate,
+                        end_date: options.endDate,
+                        page: options.page,
+                        limit: options.limit,
+                    }, this.getUserAccess());
+                    // Decrypt message content for dead letters
+                    const deadLetters = result.deadLetters.map((dl) => (Object.assign(Object.assign({}, dl), { original_message: dl.original_message ? Object.assign(Object.assign({}, dl.original_message), { message_decrypted: dl.original_message.message_encrypted
+                                ? this.decryptPayload(dl.original_message.message_encrypted)
+                                : undefined }) : dl.original_message })));
+                    return {
+                        deadLetters,
+                        total: result.total,
+                        page: result.page,
+                        limit: result.limit,
+                        hasMore: result.hasMore,
+                    };
+                }
+                catch (error) {
+                    if (error instanceof BrokerError)
+                        throw error;
+                    throw new BrokerError(`Failed to get broker dead letters: ${error}`, 'GET_DEAD_LETTERS_FAILED', error);
+                }
+            },
+            /**
+             * Get comprehensive broker message statistics
+             */
+            getStats: async (options) => {
+                try {
+                    await this.initializeProductForEvents(options.product);
+                    return await this.processorApiService.getBrokerComprehensiveStats({
+                        product_tag: options.product,
+                        env: options.env,
+                        broker_tag: options.brokerTag,
+                    }, this.getUserAccess());
+                }
+                catch (error) {
+                    if (error instanceof BrokerError)
+                        throw error;
+                    throw new BrokerError(`Failed to get broker stats: ${error}`, 'GET_STATS_FAILED', error);
+                }
+            },
+            /**
+             * Get broker dashboard overview data
+             * Decrypts message content for recent messages
+             */
+            getDashboard: async (options) => {
+                try {
+                    await this.initializeProductForEvents(options.product);
+                    const result = await this.processorApiService.getBrokerDashboard({
+                        product_tag: options.product,
+                        env: options.env,
+                        broker_tag: options.brokerTag,
+                    }, this.getUserAccess());
+                    // Decrypt recent messages
+                    const recentMessages = (result.recent_messages || []).map((msg) => (Object.assign(Object.assign({}, msg), { message_decrypted: msg.message_encrypted
+                            ? this.decryptPayload(msg.message_encrypted)
+                            : undefined })));
+                    return Object.assign(Object.assign({}, result), { recent_messages: recentMessages });
+                }
+                catch (error) {
+                    if (error instanceof BrokerError)
+                        throw error;
+                    throw new BrokerError(`Failed to get broker dashboard: ${error}`, 'GET_DASHBOARD_FAILED', error);
+                }
+            },
+        };
         this.config = config;
-        this.productBuilderService = new products_service_1.default({
+        this.productBuilderService = (_a = config.preInitializedProductBuilder) !== null && _a !== void 0 ? _a : new products_service_1.default({
             workspace_id: config.workspace_id,
             public_key: config.public_key,
             user_id: config.user_id,
             token: config.token,
             env_type: config.env_type,
             redis_client: config.redis_client,
+            access_key: config.access_key,
         });
         this.processorApiService = new processorApi_service_1.ProcessorApiService(config.env_type);
         // Initialize CacheManager if Redis client is provided
@@ -111,16 +295,40 @@ class BrokersService {
             token: this.config.token,
             workspace_id: this.config.workspace_id,
             public_key: this.config.public_key,
+            access_key: this.config.access_key,
         };
     }
     /**
-     * Initialize product and get broker configuration
+     * Initialize product and get broker configuration (with caching)
      */
-    async initializeBroker(productTag, envSlug, event) {
+    async initializeBroker(productTag, envSlug, event, message) {
+        var _a, _b;
         // Parse event to get broker and topic tags
         const { brokerTag, topicTag } = (0, broker_util_1.parseEventString)(event);
-        // Initialize product
-        await this.productBuilderService.initializeProductByTag(productTag);
+        // Check cache first (keyed by product:broker:env)
+        const cacheKey = `${productTag}:${brokerTag}:${envSlug}`;
+        const cached = this.initializedBrokerCache.get(cacheKey);
+        if (cached) {
+            // For cached brokers, just check if topic exists (might be new topic)
+            let topic = (_a = cached.broker.topics) === null || _a === void 0 ? void 0 : _a.find(t => t.tag === topicTag);
+            if (!topic) {
+                if (message) {
+                    this.ensureTopicRegistered(brokerTag, topicTag, message);
+                }
+                topic = { tag: topicTag, name: topicTag, sample: message || {} };
+            }
+            // Determine topic URL
+            let topicUrl = topic.name;
+            if (cached.brokerEnv.type === types_1.MessageBrokerTypes.AWS_SQS) {
+                topicUrl = (0, broker_util_1.getSqsQueueUrl)(topic, envSlug);
+            }
+            return Object.assign(Object.assign({}, cached), { topic, topicUrl });
+        }
+        // Initialize product only if not already done
+        if (!this.initializedProducts.has(productTag)) {
+            await this.productBuilderService.initializeProductByTag(productTag);
+            this.initializedProducts.add(productTag);
+        }
         this.productTag = productTag;
         this.productId = this.productBuilderService.fetchProductId() || '';
         this.privateKey = this.productBuilderService.fetchPrivateKey();
@@ -129,15 +337,21 @@ class BrokersService {
         if (!broker) {
             throw new BrokerError(`Message Broker ${brokerTag} not found`, 'BROKER_NOT_FOUND');
         }
+        if (!broker.envs || !Array.isArray(broker.envs)) {
+            throw new BrokerError(`Message Broker ${brokerTag} has no environments configured`, 'BROKER_NO_ENVS');
+        }
         // Get environment-specific broker config
         const brokerEnv = broker.envs.find((el) => el.slug === envSlug);
         if (!brokerEnv) {
             throw new BrokerError(`Broker environment for ${envSlug} not found`, 'BROKER_ENV_NOT_FOUND');
         }
-        // Fetch topic configuration
-        const topic = await this.productBuilderService.fetchMessageBrokerTopic(event);
+        // Check if topic exists in broker's topics array
+        let topic = (_b = broker.topics) === null || _b === void 0 ? void 0 : _b.find(t => t.tag === topicTag);
         if (!topic) {
-            throw new BrokerError(`Topic ${topicTag} not found in broker ${brokerTag}`, 'TOPIC_NOT_FOUND');
+            if (message) {
+                this.ensureTopicRegistered(brokerTag, topicTag, message);
+            }
+            topic = { tag: topicTag, name: topicTag, sample: message || {} };
         }
         // Resolve any secret references in broker config
         let resolvedConfig = brokerEnv.config;
@@ -148,18 +362,23 @@ class BrokersService {
                 resolvedConfig = resolved.value;
             }
             else {
-                throw new BrokerError(`Broker configuration contains secret references but secrets service is not initialized. ` +
-                    `Please ensure the Ductape client is properly initialized.`, 'SECRETS_NOT_INITIALIZED');
+                throw new BrokerError(`Broker configuration contains secret references but secrets service is not initialized.`, 'SECRETS_NOT_INITIALIZED');
             }
         }
-        // Create a new brokerEnv object with resolved config
+        // Create resolved broker env
         const resolvedBrokerEnv = Object.assign(Object.assign({}, brokerEnv), { config: resolvedConfig });
         // Determine the topic URL/name
         let topicUrl = topic.name;
-        // Special handling for AWS SQS which requires queue URLs
         if (brokerEnv.type === types_1.MessageBrokerTypes.AWS_SQS) {
             topicUrl = (0, broker_util_1.getSqsQueueUrl)(topic, envSlug);
         }
+        // Cache the result (without topic, as topics may vary per call)
+        this.initializedBrokerCache.set(cacheKey, {
+            broker,
+            topic, // Store last topic, but will be re-evaluated on cache hit
+            brokerEnv: resolvedBrokerEnv,
+            topicUrl,
+        });
         return { broker, topic, brokerEnv: resolvedBrokerEnv, topicUrl };
     }
     /**
@@ -179,26 +398,109 @@ class BrokersService {
     }
     /**
      * Validate cache tag exists in product and return cache configuration
+     * Uses local in-memory cache to avoid repeated API calls (5 minute TTL)
      */
     async validateCache(cacheTag) {
+        const cacheKey = `${this.productTag}:${cacheTag}`;
+        const cached = this.cacheConfigCache.get(cacheKey);
+        const now = Date.now();
+        const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+        // Return cached config if still valid
+        if (cached && (now - cached.fetchedAt) < CACHE_TTL) {
+            return { expiry: cached.expiry };
+        }
+        // Fetch from API and cache the result
         const cache = await this.productBuilderService.fetchCache(cacheTag);
         if (!cache) {
             throw new BrokerError(`Cache configuration '${cacheTag}' does not exist`, 'CACHE_NOT_FOUND');
         }
+        this.cacheConfigCache.set(cacheKey, { expiry: cache.expiry, fetchedAt: now });
         return { expiry: cache.expiry };
     }
     /**
-     * Get or create broker service instance
+     * Generate a cache key for the broker service pool
+     */
+    getBrokerPoolKey(type, config) {
+        var _a;
+        // Use type + url/host as the key (url for RabbitMQ, host for others)
+        const configKey = (config === null || config === void 0 ? void 0 : config.url) || (config === null || config === void 0 ? void 0 : config.host) || ((_a = config === null || config === void 0 ? void 0 : config.brokers) === null || _a === void 0 ? void 0 : _a[0]) || JSON.stringify(config);
+        return `${type}:${configKey}`;
+    }
+    /**
+     * Get or create broker service instance (with connection pooling)
      */
     async getBrokerService(type, config) {
         if ((0, broker_util_1.isBrowser)()) {
             throw new BrokerError('Broker operations are not supported in browser environment', 'BROWSER_NOT_SUPPORTED');
         }
+        // Check if we have a cached connection
+        const poolKey = this.getBrokerPoolKey(type, config);
+        const existingService = this.brokerServicePool.get(poolKey);
+        if (existingService) {
+            return existingService;
+        }
         const createBrokerService = await (0, broker_util_1.loadBrokerService)();
         if (!createBrokerService) {
             throw new BrokerError('Failed to load broker service', 'BROKER_SERVICE_LOAD_FAILED');
         }
-        return createBrokerService(type, config);
+        const service = createBrokerService(type, config);
+        // Store in pool for reuse
+        this.brokerServicePool.set(poolKey, service);
+        return service;
+    }
+    /**
+     * Disconnect all pooled broker connections (e.g. at end of workflow run). Safe if pool is empty.
+     */
+    async disconnectAll() {
+        for (const service of this.brokerServicePool.values()) {
+            try {
+                if (typeof service.disconnect === 'function') {
+                    await service.disconnect();
+                }
+            }
+            catch (_a) {
+                // Non-fatal
+            }
+        }
+        this.brokerServicePool.clear();
+    }
+    /**
+     * Pre-warm broker connections for the given product/env and broker tags.
+     * Call before step execution so the first produce step reuses connections. Same auth and secrets as publish.
+     */
+    async warmBrokerConnections(options) {
+        const { product, env, brokerTags } = options;
+        if (brokerTags.length === 0)
+            return;
+        for (const brokerTag of brokerTags) {
+            try {
+                const { brokerEnv } = await this.initializeBroker(product, env, `${brokerTag}:_warm`, {});
+                const service = await this.getBrokerService(brokerEnv.type, brokerEnv.config);
+                if (typeof service.connect === 'function')
+                    await service.connect();
+            }
+            catch (_a) {
+                // Non-fatal: step will connect on first use
+            }
+        }
+    }
+    /**
+     * Auto-register a topic in the background (fire-and-forget)
+     * This is intentionally lightweight - no pre-check, no post-fetch
+     */
+    ensureTopicRegistered(brokerTag, topicTag, message) {
+        const fullTag = `${brokerTag}:${topicTag}`;
+        // Fire-and-forget: don't await, let it run in background
+        this.productBuilderService.createMessageBrokerTopic({
+            tag: fullTag,
+            name: topicTag,
+            description: `Auto-registered topic for ${fullTag}`,
+            sample: JSON.stringify(message || {}),
+            idempotent: false,
+        }).catch(() => {
+            // Silently ignore - topic might already exist or registration failed
+            // Either way, the publish should proceed
+        });
     }
     /**
      * Auto-register a producer if it doesn't exist
@@ -244,15 +546,16 @@ class BrokersService {
      * Publish a message to a broker topic
      */
     async publish(options) {
-        var _a, _b;
+        var _a, _b, _c, _d;
         const process_id = (0, processor_utils_1.generateObjectId)();
         const start = Date.now();
         const { cache } = options;
         let sessionLogFields = {};
         let resolvedMessage = options.message;
+        // Parse event string early to have broker/topic tags available for error logging
+        const { brokerTag, topicTag } = (0, broker_util_1.parseEventString)(options.event);
         try {
-            const { broker, topic, brokerEnv, topicUrl } = await this.initializeBroker(options.product, options.env, options.event);
-            const { brokerTag, topicTag } = (0, broker_util_1.parseEventString)(options.event);
+            const { broker, topic, brokerEnv, topicUrl } = await this.initializeBroker(options.product, options.env, options.event, options.message);
             // Store private key for cache operations
             if (this.privateKey) {
                 this.privateKeys.set(options.product, this.privateKey);
@@ -261,7 +564,12 @@ class BrokersService {
             if (options.session && this.privateKey) {
                 // Convert session object { tag, token } to string format "tag:token"
                 const sessionToken = `${options.session.tag}:${options.session.token}`;
-                const sessionResult = await (0, sessions_1.processSessionForExecution)(sessionToken, this.privateKey, { message: options.message }, options.env);
+                const sessionResult = await (0, sessions_1.processSessionForExecution)(sessionToken, this.privateKey, { message: options.message }, options.env, {
+                    fetchSessionSelector: async (sessionTag) => {
+                        const sessionConfig = await this.productBuilderService.fetchSession(sessionTag);
+                        return sessionConfig === null || sessionConfig === void 0 ? void 0 : sessionConfig.selector;
+                    },
+                });
                 if (sessionResult.error) {
                     throw new BrokerError(`Session validation failed: ${sessionResult.error}`, 'SESSION_INVALID');
                 }
@@ -271,7 +579,7 @@ class BrokersService {
             // Auto-register producer if not exists
             await this.ensureProducerRegistered(brokerTag, topicTag, options.producer);
             this.initializeLogService();
-            const baseLogs = Object.assign({ product_tag: options.product, product_id: this.productId, workspace_id: this.config.workspace_id, env: options.env, process_id, type: types_2.LogEventTypes.MESSAGEBROKER, parent_tag: brokerTag, child_tag: topicTag, data: { event: options.event } }, sessionLogFields);
+            const baseLogs = Object.assign({ product_tag: options.product, product_id: this.productId, workspace_id: this.config.workspace_id, env: options.env, process_id, type: types_2.LogEventTypes.PRODUCER, parent_tag: brokerTag, child_tag: topicTag, data: { event: options.event } }, sessionLogFields);
             this.logService.add(Object.assign(Object.assign({}, baseLogs), { start, message: 'Publish to broker - initiated', status: types_2.LogEventStatus.PROCESSING }));
             // Check cache for idempotency (prevent duplicate publishes with same data)
             if (cache && this.cacheManager && this.privateKey) {
@@ -290,22 +598,47 @@ class BrokersService {
                 if (cached.hit && cached.data) {
                     const end = Date.now();
                     this.logService.add(Object.assign(Object.assign({}, baseLogs), { start,
-                        end, message: `Publish to broker - cache hit (${cached.source})`, successful_execution: true, status: types_2.LogEventStatus.SUCCESS, data: { event: options.event, cache_source: cached.source } }));
+                        end, message: `Publish to broker - cache hit (${cached.source})`, successful_execution: true, status: types_2.LogEventStatus.SUCCESS, data: { event: options.event, cache_source: cached.source }, cache_tag: cache, cache_key: cached.key, cache_status: true }));
                     this.logService.publish();
                     return cached.data;
                 }
             }
+            // Generate message_id client-side and track asynchronously (fire-and-forget)
+            const producerTag = ((_a = options.producer) === null || _a === void 0 ? void 0 : _a.tag) || `producer-${brokerTag}-${topicTag}`;
+            const messageId = (0, processor_utils_1.generateObjectId)();
+            // Encrypt message on SDK side - NEVER send private_key over HTTP
+            const messageEncrypted = (0, processor_utils_1.encrypt)(JSON.stringify(resolvedMessage), this.privateKey);
+            // Track the message in the database (fire-and-forget - no await)
+            this.processorApiService.trackBrokerMessage({
+                workspace_id: this.config.workspace_id,
+                product_id: this.productId,
+                product_tag: options.product,
+                env: options.env,
+                broker_tag: brokerTag,
+                topic_tag: topicTag,
+                event: options.event,
+                producer_tag: producerTag,
+                message_encrypted: messageEncrypted,
+                process_id,
+                session_tag: (_b = options.session) === null || _b === void 0 ? void 0 : _b.tag,
+                metadata: { cache: options.cache },
+                message_id: messageId,
+            }, this.getUserAccess()).catch(() => {
+                // Silently ignore tracking errors - non-blocking
+            });
+            // Add the message_id to the message so consumers can track it
+            const messageWithTracking = Object.assign(Object.assign({}, resolvedMessage), { __ductape_message_id: messageId });
             // Get broker service and publish
             const brokerService = await this.getBrokerService(brokerEnv.type, brokerEnv.config);
-            await brokerService.publish(topicUrl, resolvedMessage);
+            await brokerService.publish(topicUrl, messageWithTracking);
             const result = {
                 success: true,
                 process_id,
             };
-            // Store in cache for idempotency
+            // Store in cache for idempotency (fire-and-forget)
             if (cache && this.cacheManager && this.privateKey) {
                 const cacheConfig = await this.validateCache(cache);
-                await this.cacheManager.store({
+                this.cacheManager.store({
                     cache_tag: cache,
                     product_tag: options.product,
                     component_tag: `${brokerTag}:${topicTag}`,
@@ -317,18 +650,26 @@ class BrokersService {
                 }, result);
             }
             const end = Date.now();
-            this.logService.add(Object.assign(Object.assign({}, baseLogs), { start,
-                end, message: 'Publish to broker - success', successful_execution: true, status: types_2.LogEventStatus.SUCCESS }));
+            const successLogKey = `${options.product}:${options.env}:${options.event}`;
+            const now = Date.now();
+            const isDuplicateSuccessLog = this.lastPublishSuccessLogKey === successLogKey &&
+                now - this.lastPublishSuccessLogTime < BrokersService.PUBLISH_SUCCESS_LOG_DEDUPE_MS;
+            if (!isDuplicateSuccessLog) {
+                this.logService.add(Object.assign(Object.assign({}, baseLogs), { start,
+                    end, message: 'Publish to broker - success', successful_execution: true, status: types_2.LogEventStatus.SUCCESS }));
+                this.lastPublishSuccessLogKey = successLogKey;
+                this.lastPublishSuccessLogTime = now;
+            }
             this.logService.publish();
             return result;
         }
         catch (error) {
             const end = Date.now();
             this.initializeLogService();
-            (_a = this.logService) === null || _a === void 0 ? void 0 : _a.add(Object.assign({ product_tag: options.product, workspace_id: this.config.workspace_id, env: options.env, process_id,
+            (_c = this.logService) === null || _c === void 0 ? void 0 : _c.add(Object.assign({ product_tag: options.product, workspace_id: this.config.workspace_id, env: options.env, process_id,
                 start,
-                end, type: types_2.LogEventTypes.MESSAGEBROKER, message: 'Publish to broker - failed', failed_execution: true, data: { event: options.event, error: String(error) }, status: types_2.LogEventStatus.FAIL }, sessionLogFields));
-            await ((_b = this.logService) === null || _b === void 0 ? void 0 : _b.publish());
+                end, type: types_2.LogEventTypes.PRODUCER, parent_tag: brokerTag, child_tag: topicTag, message: 'Publish to broker - failed', failed_execution: true, data: { event: options.event, error: String(error) }, status: types_2.LogEventStatus.FAIL }, sessionLogFields));
+            await ((_d = this.logService) === null || _d === void 0 ? void 0 : _d.publish());
             if (error instanceof BrokerError) {
                 throw error;
             }
@@ -339,11 +680,12 @@ class BrokersService {
      * Subscribe to a broker topic
      */
     async subscribe(options) {
-        var _a, _b;
+        var _a, _b, _c;
         const process_id = (0, processor_utils_1.generateObjectId)();
         const start = Date.now();
         try {
-            const { broker, topic, brokerEnv, topicUrl } = await this.initializeBroker(options.product, options.env, options.event);
+            // For subscribe, we don't auto-register topics - they should already exist
+            const { broker, topic, brokerEnv, topicUrl } = await this.initializeBroker(options.product, options.env, options.event, undefined);
             const { brokerTag, topicTag } = (0, broker_util_1.parseEventString)(options.event);
             // Auto-register consumer if not exists
             await this.ensureConsumerRegistered(brokerTag, topicTag, options.consumer);
@@ -354,15 +696,80 @@ class BrokersService {
                 workspace_id: this.config.workspace_id,
                 env: options.env,
                 process_id,
-                type: types_2.LogEventTypes.MESSAGEBROKER,
+                type: types_2.LogEventTypes.CONSUMER,
                 parent_tag: brokerTag,
                 child_tag: topicTag,
                 data: { event: options.event },
             };
             this.logService.add(Object.assign(Object.assign({}, baseLogs), { start, message: 'Subscribe to broker - initiated', status: types_2.LogEventStatus.PROCESSING }));
-            // Get broker service and subscribe
+            // Get broker service and subscribe with message tracking wrapper
             const brokerService = await this.getBrokerService(brokerEnv.type, brokerEnv.config);
-            await brokerService.subscribe(topicUrl, options.callback);
+            const consumerTag = ((_a = options.consumer) === null || _a === void 0 ? void 0 : _a.tag) || `consumer-${brokerTag}-${topicTag}`;
+            // Pre-cache user access for callback use (avoid repeated calls)
+            const userAccess = this.getUserAccess();
+            // Wrap the callback to track message consumption (optimized for low latency)
+            const wrappedCallback = async (message) => {
+                const messageReceiveTime = Date.now();
+                const consumeProcessId = (0, processor_utils_1.generateObjectId)();
+                // Extract message_id from the message
+                const msgWithId = message;
+                const messageId = msgWithId.__ductape_message_id;
+                // Execute user callback FIRST - this is the critical path
+                let callbackSuccess = true;
+                let callbackError;
+                try {
+                    await options.callback(message);
+                }
+                catch (err) {
+                    callbackSuccess = false;
+                    callbackError = err;
+                }
+                const callbackEnd = Date.now();
+                // Fire-and-forget: All tracking/logging happens async after callback completes
+                // This ensures user callback latency is not affected by our tracking overhead
+                setImmediate(() => {
+                    var _a, _b;
+                    const consumeBaseLogs = {
+                        product_tag: options.product,
+                        product_id: this.productId,
+                        workspace_id: this.config.workspace_id,
+                        env: options.env,
+                        process_id: consumeProcessId,
+                        type: types_2.LogEventTypes.CONSUMER,
+                        parent_tag: brokerTag,
+                        child_tag: topicTag,
+                        data: { event: options.event, message_id: messageId, consumer_tag: consumerTag },
+                    };
+                    if (messageId) {
+                        // Calculate actual callback processing time
+                        const processingTime = callbackEnd - messageReceiveTime;
+                        // Register consumer first, then update status (sequential to avoid race condition)
+                        this.processorApiService.registerBrokerConsumer({
+                            message_id: messageId,
+                            consumer_tag: consumerTag,
+                        }, userAccess).then(() => {
+                            // Now update the status with processing time
+                            return this.processorApiService.updateBrokerConsumerStatus({
+                                message_id: messageId,
+                                consumer_tag: consumerTag,
+                                status: callbackSuccess ? 'success' : 'failed',
+                                error: callbackError ? String(callbackError) : undefined,
+                                processing_time: processingTime,
+                            }, userAccess);
+                        }).catch(() => { }); // Silently ignore tracking errors
+                    }
+                    // Log consumption (fire-and-forget)
+                    (_a = this.logService) === null || _a === void 0 ? void 0 : _a.add(Object.assign(Object.assign(Object.assign({}, consumeBaseLogs), { start: messageReceiveTime, end: callbackEnd, message: messageId
+                            ? (callbackSuccess ? 'Message consumed - success' : 'Message consumed - failed')
+                            : 'Message consumed - untracked (no message_id)', successful_execution: callbackSuccess, failed_execution: !callbackSuccess, status: callbackSuccess ? types_2.LogEventStatus.SUCCESS : types_2.LogEventStatus.FAIL }), (callbackError && { data: Object.assign(Object.assign({}, consumeBaseLogs.data), { error: String(callbackError) }) })));
+                    (_b = this.logService) === null || _b === void 0 ? void 0 : _b.publish();
+                });
+                // Re-throw callback error if any (so RabbitMQ can handle retry/nack)
+                if (callbackError) {
+                    throw callbackError;
+                }
+            };
+            await brokerService.subscribe(topicUrl, wrappedCallback);
             const end = Date.now();
             this.logService.add(Object.assign(Object.assign({}, baseLogs), { start,
                 end, message: 'Subscribe to broker - success', successful_execution: true, status: types_2.LogEventStatus.SUCCESS }));
@@ -375,20 +782,20 @@ class BrokersService {
         catch (error) {
             const end = Date.now();
             this.initializeLogService();
-            (_a = this.logService) === null || _a === void 0 ? void 0 : _a.add({
+            (_b = this.logService) === null || _b === void 0 ? void 0 : _b.add({
                 product_tag: options.product,
                 workspace_id: this.config.workspace_id,
                 env: options.env,
                 process_id,
                 start,
                 end,
-                type: types_2.LogEventTypes.MESSAGEBROKER,
+                type: types_2.LogEventTypes.CONSUMER,
                 message: 'Subscribe to broker - failed',
                 failed_execution: true,
                 data: { event: options.event, error: String(error) },
                 status: types_2.LogEventStatus.FAIL,
             });
-            await ((_b = this.logService) === null || _b === void 0 ? void 0 : _b.publish());
+            await ((_c = this.logService) === null || _c === void 0 ? void 0 : _c.publish());
             if (error instanceof BrokerError) {
                 throw error;
             }
@@ -793,5 +1200,6 @@ class BrokersService {
     }
 }
 exports.BrokersService = BrokersService;
+BrokersService.PUBLISH_SUCCESS_LOG_DEDUPE_MS = 2000;
 exports.default = BrokersService;
 //# sourceMappingURL=brokers.service.js.map