@hotmeshio/hotmesh 0.5.3 → 0.5.4

package/build/services/connector/providers/postgres.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgresConnection = void 0;
 const __1 = require("..");
+const utils_1 = require("../../../modules/utils");
 class PostgresConnection extends __1.AbstractConnection {
     constructor() {
         super(...arguments);
@@ -15,6 +16,54 @@ class PostgresConnection extends __1.AbstractConnection {
             idleTimeoutMillis: 30000,
         };
     }
+    /**
+     * Get comprehensive connection statistics for monitoring taskQueue pooling effectiveness
+     */
+    static getConnectionStats() {
+        const taskQueueDetails = Array.from(this.taskQueueConnections.entries()).map(([key, connection]) => ({
+            key,
+            connectionId: connection.getConnectionId() || 'unknown',
+            reusedCount: connection.reusedCount || 0
+        }));
+        return {
+            totalPoolClients: this.poolClientInstances.size,
+            totalConnections: this.connectionInstances.size,
+            taskQueueConnections: this.taskQueueConnections.size,
+            taskQueueDetails
+        };
+    }
+    /**
+     * Log current connection statistics - useful for debugging connection pooling
+     */
+    static logConnectionStats(logger) {
+        const stats = this.getConnectionStats();
+        const message = `PostgreSQL Connection Stats: ${stats.totalConnections} total connections, ${stats.taskQueueConnections} taskQueue pools, ${stats.totalPoolClients} pool clients`;
+        if (logger) {
+            logger.info('postgres-connection-stats', {
+                ...stats,
+                message
+            });
+        }
+        else {
+            console.log(message, stats);
+        }
+    }
+    /**
+     * Check taskQueue pooling effectiveness - returns metrics about connection reuse
+     */
+    static getPoolingEffectiveness() {
+        const stats = this.getConnectionStats();
+        const totalReuses = stats.taskQueueDetails.reduce((sum, detail) => sum + detail.reusedCount, 0);
+        const averageReusesPerPool = stats.taskQueueConnections > 0 ? totalReuses / stats.taskQueueConnections : 0;
+        const poolingEfficiency = stats.totalConnections > 0 ? (stats.taskQueueConnections / stats.totalConnections) * 100 : 0;
+        return {
+            totalConnections: stats.totalConnections,
+            taskQueuePools: stats.taskQueueConnections,
+            totalReuses,
+            averageReusesPerPool: Math.round(averageReusesPerPool * 100) / 100,
+            poolingEfficiency: Math.round(poolingEfficiency * 100) / 100,
+        };
+    }
     async createConnection(clientConstructor, options, config = {}) {
         try {
             let connection;
@@ -55,11 +104,20 @@ class PostgresConnection extends __1.AbstractConnection {
         }
         return this.poolClientInstance || this.connection;
     }
+    /**
+     * Get the connection ID for monitoring purposes
+     */
+    getConnectionId() {
+        return this.id;
+    }
     static async disconnectAll() {
+        //log stats
+        //this.logConnectionStats();
         if (!this.disconnecting) {
             this.disconnecting = true;
             await this.disconnectPoolClients();
             await this.disconnectConnections();
+            this.taskQueueConnections.clear();
             this.disconnecting = false;
         }
     }
@@ -81,6 +139,53 @@ class PostgresConnection extends __1.AbstractConnection {
     static isPoolClient(client) {
         return !(isNaN(client?.totalCount) && isNaN(client?.idleCount));
     }
+    /**
+     * Creates a taskQueue-based connection key for connection pooling.
+     * This allows multiple providers (store, sub, stream) to reuse the same connection
+     * when they share the same taskQueue and database configuration.
+     */
+    static createTaskQueueConnectionKey(taskQueue, options, provider) {
+        const configHash = (0, utils_1.hashOptions)(options);
+        const providerType = provider?.split('.')[0] || 'postgres';
+        return `${providerType}:${taskQueue || 'default'}:${configHash}`;
+    }
+    /**
+     * Gets or creates a PostgreSQL connection based on taskQueue and database configuration.
+     * If a connection already exists for the same taskQueue + config, it will be reused.
+     * This optimization reduces connection overhead for PostgreSQL providers.
+     */
+    static async getOrCreateTaskQueueConnection(id, taskQueue, clientConstructor, options, config = {}) {
+        // Only use taskQueue pooling for PostgreSQL providers
+        if (!taskQueue || !config.provider?.startsWith('postgres')) {
+            return await this.connect(id, clientConstructor, options, config);
+        }
+        const connectionKey = this.createTaskQueueConnectionKey(taskQueue, options, config.provider);
+        // Check if we already have a connection for this taskQueue + config
+        if (this.taskQueueConnections.has(connectionKey)) {
+            const existingConnection = this.taskQueueConnections.get(connectionKey);
+            // Track reuse count for monitoring
+            existingConnection.reusedCount = (existingConnection.reusedCount || 0) + 1;
+            this.logger.debug('postgres-connection-reused', {
+                connectionKey,
+                taskQueue,
+                originalId: existingConnection.id,
+                newId: id,
+                reuseCount: existingConnection.reusedCount,
+            });
+            return existingConnection;
+        }
+        // Create new connection and cache it for the taskQueue
+        const newConnection = await this.connect(id, clientConstructor, options, config);
+        // Initialize reuse tracking
+        newConnection.reusedCount = 0;
+        this.taskQueueConnections.set(connectionKey, newConnection);
+        this.logger.debug('postgres-connection-created-for-taskqueue', {
+            connectionKey,
+            taskQueue,
+            connectionId: id,
+        });
+        return newConnection;
+    }
     static async getTransactionClient(transactionClient) {
         let client;
         let type;
@@ -100,3 +205,5 @@ exports.PostgresConnection = PostgresConnection;
 PostgresConnection.poolClientInstances = new Set();
 //statically track all connections (//call 'end')
 PostgresConnection.connectionInstances = new Set();
+//track connections by taskQueue + database config for reuse
+PostgresConnection.taskQueueConnections = new Map();

package/build/services/hotmesh/index.d.ts

@@ -264,6 +264,14 @@ declare class HotMesh {
      * @private
      */
     doWork(config: HotMeshConfig, logger: ILogger): Promise<void>;
+    /**
+     * Initialize task queue with proper precedence:
+     * 1. Use component-specific queue if set (engine/worker)
+     * 2. Use global config queue if set
+     * 3. Use default queue as fallback
+     * @private
+     */
+    private initTaskQueue;
     /**
      * Starts a workflow
      * @example

package/build/services/hotmesh/index.js

@@ -275,6 +275,8 @@ class HotMesh {
             if (config.engine.connection.readonly) {
                 config.engine.readonly = true;
             }
+            // Initialize task queue for engine
+            config.engine.taskQueue = this.initTaskQueue(config.engine.taskQueue, config.taskQueue);
             await factory_1.ConnectorService.initClients(config.engine);
             this.engine = await engine_1.EngineService.init(this.namespace, this.appId, this.guid, config, logger);
         }
@@ -308,8 +310,33 @@ class HotMesh {
      * @private
      */
     async doWork(config, logger) {
+        // Initialize task queues for workers
+        if (config.workers) {
+            for (const worker of config.workers) {
+                worker.taskQueue = this.initTaskQueue(worker.taskQueue, config.taskQueue);
+            }
+        }
         this.workers = await worker_1.WorkerService.init(this.namespace, this.appId, this.guid, config, logger);
     }
+    /**
+     * Initialize task queue with proper precedence:
+     * 1. Use component-specific queue if set (engine/worker)
+     * 2. Use global config queue if set
+     * 3. Use default queue as fallback
+     * @private
+     */
+    initTaskQueue(componentQueue, globalQueue) {
+        // Component-specific queue takes precedence
+        if (componentQueue) {
+            return componentQueue;
+        }
+        // Global config queue is next
+        if (globalQueue) {
+            return globalQueue;
+        }
+        // Default queue as fallback
+        return enums_1.DEFAULT_TASK_QUEUE;
+    }
     // ************* PUB/SUB METHODS *************
     /**
      * Starts a workflow

package/build/services/memflow/client.d.ts

@@ -60,7 +60,7 @@ export declare class ClientService {
     /**
      * @private
      */
-    getHotMeshClient: (workflowTopic: string | null, namespace?: string) => Promise<HotMesh>;
+    getHotMeshClient: (taskQueue: string | null, namespace?: string) => Promise<HotMesh>;
     /**
      * Creates a stream where messages can be published to ensure there is a
      * channel in place when the message arrives (a race condition for those

package/build/services/memflow/client.js

@@ -54,7 +54,7 @@ class ClientService {
         /**
          * @private
          */
-        this.getHotMeshClient = async (workflowTopic, namespace) => {
+        this.getHotMeshClient = async (taskQueue, namespace) => {
             //namespace isolation requires the connection options to be hashed
             //as multiple intersecting databases can be used by the same service
             //hashing options allows for reuse of the same connection without risk of
@@ -71,6 +71,7 @@ class ClientService {
             const readonly = this.connection.readonly ?? undefined;
             let hotMeshClient = hotmesh_1.HotMesh.init({
                 appId: targetNS,
+                taskQueue,
                 logLevel: enums_1.HMSH_LOGLEVEL,
                 engine: {
                     readonly,
@@ -130,7 +131,7 @@ class ClientService {
                 const spn = options.workflowSpan;
                 //hotmesh `topic` is equivalent to `queue+workflowname` pattern in other systems
                 const workflowTopic = `${taskQueueName}-${workflowName}`;
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
+                const hotMeshClient = await this.getHotMeshClient(taskQueueName, options.namespace);
                 //verify that the stream channel exists before enqueueing
                 await this.verifyStream(hotMeshClient, workflowTopic, options.namespace);
                 const payload = {
@@ -182,7 +183,8 @@ class ClientService {
              * ```
              */
             hook: async (options) => {
-                const workflowTopic = `${options.taskQueue ?? options.entity}-${options.entity ?? options.workflowName}`;
+                const taskQueue = options.taskQueue ?? options.entity;
+                const workflowTopic = `${taskQueue}-${options.entity ?? options.workflowName}`;
                 const payload = {
                     arguments: [...options.args],
                     id: options.workflowId,
@@ -192,7 +194,7 @@ class ClientService {
                     maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_MEMFLOW_MAX_INTERVAL),
                 };
                 //seed search data before entering
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
+                const hotMeshClient = await this.getHotMeshClient(taskQueue, options.namespace);
                 const msgId = await hotMeshClient.hook(`${hotMeshClient.appId}.flow.signal`, payload, types_1.StreamStatus.PENDING, 202);
                 //todo: commit search data BEFORE enqueuing hook
                 if (options.search?.data) {
@@ -222,7 +224,7 @@ class ClientService {
              */
             getHandle: async (taskQueue, workflowName, workflowId, namespace) => {
                 const workflowTopic = `${taskQueue}-${workflowName}`;
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
+                const hotMeshClient = await this.getHotMeshClient(taskQueue, namespace);
                 return new handle_1.WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
             },
             /**
@@ -249,7 +251,7 @@ class ClientService {
              */
             search: async (taskQueue, workflowName, namespace, index, ...query) => {
                 const workflowTopic = `${taskQueue}-${workflowName}`;
-                const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
+                const hotMeshClient = await this.getHotMeshClient(taskQueue, namespace);
                 try {
                     return await this.search(hotMeshClient, index, query);
                 }

package/build/services/memflow/worker.js

@@ -174,6 +174,7 @@ class WorkerService {
         const targetTopic = `${optionsHash}.${targetNamespace}.${activityTopic}`;
         const hotMeshWorker = await hotmesh_1.HotMesh.init({
             guid: config.guid ? `${config.guid}XA` : undefined,
+            taskQueue: config.taskQueue,
             logLevel: config.options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
             appId: targetNamespace,
             engine: { connection: providerConfig },
@@ -258,6 +259,7 @@ class WorkerService {
         const targetTopic = `${optionsHash}.${targetNamespace}.${workflowTopic}`;
         const hotMeshWorker = await hotmesh_1.HotMesh.init({
             guid: config.guid,
+            taskQueue: config.taskQueue,
             logLevel: config.options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
             appId: config.namespace ?? factory_1.APP_ID,
             engine: { connection: providerConfig },
@@ -518,6 +520,7 @@ WorkerService.getHotMesh = async (workflowTopic, config, options) => {
     const hotMeshClient = hotmesh_1.HotMesh.init({
         logLevel: options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
         appId: targetNamespace,
+        taskQueue: config.taskQueue,
         engine: {
             connection: { ...config?.connection },
         },

package/build/services/pipe/functions/cron.js

@@ -37,4 +37,4 @@ class CronHandler {
     }
 }
 exports.CronHandler = CronHandler;
-CronHandler.logger = new logger_1.LoggerService('hotmesh', 'meshos');
+CronHandler.logger = new logger_1.LoggerService('hotmesh', 'cron');

package/build/services/store/providers/postgres/kvtables.js

@@ -23,15 +23,21 @@ const KVTables = (context) => ({
             client = transactionClient;
         }
         try {
-            // Acquire advisory lock
+            // First, check if tables already exist (no lock needed)
+            const tablesExist = await this.checkIfTablesExist(client, appName);
+            if (tablesExist) {
+                // Tables already exist, no need to acquire lock or create tables
+                return;
+            }
+            // Tables don't exist, need to acquire lock and create them
             const lockId = this.getAdvisoryLockId(appName);
             const lockResult = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [lockId]);
             if (lockResult.rows[0].locked) {
                 // Begin transaction
                 await client.query('BEGIN');
-                // Check and create tables
-                const tablesExist = await this.checkIfTablesExist(client, appName);
-                if (!tablesExist) {
+                // Double-check tables don't exist (race condition safety)
+                const tablesStillMissing = !(await this.checkIfTablesExist(client, appName));
+                if (tablesStillMissing) {
                     await this.createTables(client, appName);
                 }
                 // Commit transaction
@@ -90,11 +96,18 @@ const KVTables = (context) => ({
                 client = transactionClient;
             }
             try {
+                // Check if tables exist directly (most efficient check)
+                const tablesExist = await this.checkIfTablesExist(client, appName);
+                if (tablesExist) {
+                    // Tables now exist, deployment is complete
+                    return;
+                }
+                // Fallback: check if the lock has been released (indicates completion)
                 const lockCheck = await client.query("SELECT NOT EXISTS (SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND objid = $1::bigint) AS unlocked", [lockId]);
                 if (lockCheck.rows[0].unlocked) {
                     // Lock has been released, tables should exist now
-                    const tablesExist = await this.checkIfTablesExist(client, appName);
-                    if (tablesExist) {
+                    const tablesExistAfterLock = await this.checkIfTablesExist(client, appName);
+                    if (tablesExistAfterLock) {
                         return;
                     }
                 }

package/build/services/store/providers/postgres/postgres.js

@@ -1073,9 +1073,20 @@ class PostgresStoreService extends __1.StoreService {
         try {
             // Set up LISTEN for time notifications
             await this.pgClient.query(`LISTEN "${channelName}"`);
-            // Set up notification handler
+            // Set up notification handler with channel filtering
             this.pgClient.on('notification', (notification) => {
-                this.handleTimeNotification(notification, timeEventCallback);
+                // Only handle time notifications (channels starting with "time_hooks_")
+                // Ignore sub and stream notifications from other providers
+                if (notification.channel.startsWith('time_hooks_')) {
+                    this.handleTimeNotification(notification, timeEventCallback);
+                }
+                else {
+                    // This is likely a notification from sub or stream provider, ignore it
+                    this.logger.debug('postgres-store-ignoring-non-time-notification', {
+                        channel: notification.channel,
+                        payloadPreview: notification.payload.substring(0, 100)
+                    });
+                }
             });
             this.logger.debug('postgres-time-scout-notifications-started', {
                 appId: this.appId,

package/build/services/stream/providers/postgres/postgres.d.ts

@@ -9,15 +9,18 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
     namespace: string;
     appId: string;
     logger: ILogger;
-    private notificationConsumers;
+    private static clientNotificationConsumers;
+    private static clientNotificationHandlers;
+    private static clientFallbackPollers;
+    private instanceNotificationConsumers;
     private notificationHandlerBound;
-    private fallbackIntervalId;
     constructor(streamClient: PostgresClientType & ProviderClient, storeClient: ProviderClient, config?: StreamConfig);
     init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    private setupClientNotificationHandler;
+    private startClientFallbackPoller;
     private isNotificationsEnabled;
     private getFallbackInterval;
     private getNotificationTimeout;
-    private startFallbackPoller;
     private checkForMissedMessages;
     private handleNotification;
     private fetchAndDeliverMessages;