@hotmeshio/hotmesh 0.6.0 → 0.7.0

package/build/services/store/providers/redis/ioredis.d.ts

@@ -0,0 +1,20 @@
+import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../../types/redis';
+import { StoreInitializable } from '../store-initializable';
+import { RedisStoreBase } from './_base';
+declare class IORedisStoreService extends RedisStoreBase<RedisClientType, RedisMultiType> implements StoreInitializable {
+    constructor(storeClient: RedisClientType);
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated. this is effectiely a wrapper for
+     * `multi` but is closer to `pipeline` in terms of usage when
+     * promises are used.
+     */
+    transact(): RedisMultiType;
+    exec(...args: any[]): Promise<string | string[] | string[][]>;
+    setnxex(key: string, value: string, expireSeconds: number): Promise<boolean>;
+    hGetAllResult(result: any): any;
+    addTaskQueues(keys: string[]): Promise<void>;
+}
+export { IORedisStoreService };

package/build/services/store/providers/redis/ioredis.js

@@ -0,0 +1,190 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisStoreService = void 0;
+const key_1 = require("../../../../modules/key");
+const enums_1 = require("../../../../modules/enums");
+const _base_1 = require("./_base");
+class IORedisStoreService extends _base_1.RedisStoreBase {
+    constructor(storeClient) {
+        super(storeClient);
+        this.commands = {
+            get: 'get',
+            set: 'set',
+            setnx: 'setnx',
+            del: 'del',
+            expire: 'expire',
+            hset: 'hset',
+            hscan: 'hscan',
+            hsetnx: 'hsetnx',
+            hincrby: 'hincrby',
+            hdel: 'hdel',
+            hget: 'hget',
+            hmget: 'hmget',
+            hgetall: 'hgetall',
+            hincrbyfloat: 'hincrbyfloat',
+            zrank: 'zrank',
+            zrange: 'zrange',
+            zrangebyscore_withscores: 'zrangebyscore',
+            zrangebyscore: 'zrangebyscore',
+            zrem: 'zrem',
+            zadd: 'zadd',
+            lmove: 'lmove',
+            lpop: 'lpop',
+            lrange: 'lrange',
+            rename: 'rename',
+            rpush: 'rpush',
+            scan: 'scan',
+            xack: 'xack',
+            xdel: 'xdel',
+        };
+    }
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated. this is effectiely a wrapper for
+     * `multi` but is closer to `pipeline` in terms of usage when
+     * promises are used.
+     */
+    transact() {
+        const my = this;
+        if (enums_1.HMSH_IS_CLUSTER) {
+            const commands = [];
+            const addCommand = (command, args) => {
+                commands.push({ command, args });
+                return multiInstance;
+            };
+            const multiInstance = {
+                sendCommand(command) {
+                    return my.storeClient.sendCommand(command);
+                },
+                async exec() {
+                    if (commands.length === 0)
+                        return [];
+                    const sameKey = commands.every((cmd) => {
+                        return cmd.args[0] === commands[0].args[0];
+                    });
+                    if (sameKey) {
+                        const multi = my.storeClient.multi();
+                        commands.forEach((cmd) => multi[cmd.command](...cmd.args));
+                        const results = await multi.exec();
+                        return results.map((item) => item);
+                    }
+                    else {
+                        return Promise.all(commands.map((cmd) => my.storeClient[cmd.command](...cmd.args)));
+                    }
+                },
+                xadd(key, id, fields, message) {
+                    return addCommand('xadd', [key, id, fields, message]);
+                },
+                xack(key, group, id) {
+                    return addCommand('xack', [key, group, id]);
+                },
+                xdel(key, id) {
+                    return addCommand('xdel', [key, id]);
+                },
+                xlen(key) {
+                    return addCommand('xlen', [key]);
+                },
+                xpending(key, group, start, end, count, consumer) {
+                    return addCommand('xpending', [
+                        key,
+                        group,
+                        start,
+                        end,
+                        count,
+                        consumer,
+                    ]);
+                },
+                xclaim(key, group, consumer, minIdleTime, id, ...args) {
+                    return addCommand('xclaim', [
+                        key,
+                        group,
+                        consumer,
+                        minIdleTime,
+                        id,
+                        ...args,
+                    ]);
+                },
+                del(key) {
+                    return addCommand('del', [key]);
+                },
+                expire: function (key, seconds) {
+                    return addCommand('expire', [key, seconds]);
+                },
+                hdel(key, itemId) {
+                    return addCommand('hdel', [key, itemId]);
+                },
+                hget(key, itemId) {
+                    return addCommand('hget', [key, itemId]);
+                },
+                hgetall(key) {
+                    return addCommand('hgetall', [key]);
+                },
+                hincrbyfloat(key, itemId, value) {
+                    return addCommand('hincrbyfloat', [key, itemId, value]);
+                },
+                hmget(key, itemIds) {
+                    return addCommand('hmget', [key, itemIds]);
+                },
+                hset(key, values) {
+                    return addCommand('hset', [key, values]);
+                },
+                lrange(key, start, end) {
+                    return addCommand('lrange', [key, start, end]);
+                },
+                rpush(key, value) {
+                    return addCommand('rpush', [key, value]);
+                },
+                zadd(...args) {
+                    return addCommand('zadd', args);
+                },
+                xgroup(command, key, groupName, id, mkStream) {
+                    return addCommand('xgroup', [command, key, groupName, id, mkStream]);
+                },
+            };
+            return multiInstance;
+        }
+        return this.storeClient.multi();
+    }
+    async exec(...args) {
+        try {
+            const response = await this.storeClient.call.apply(this.storeClient, args);
+            if (typeof response === 'string') {
+                return response;
+            }
+            else if (Array.isArray(response)) {
+                if (Array.isArray(response[0])) {
+                    return response;
+                }
+                return response;
+            }
+            return response;
+        }
+        catch (error) {
+            // Connection closed during test cleanup - log and return empty response
+            if (error?.message?.includes('Connection is closed')) {
+                return [];
+            }
+            // Re-throw unexpected errors
+            throw error;
+        }
+    }
+    async setnxex(key, value, expireSeconds) {
+        const status = await this.storeClient[this.commands.set](key, value, 'NX', 'EX', expireSeconds.toString());
+        return this.isSuccessful(status);
+    }
+    hGetAllResult(result) {
+        //ioredis response signature is [null, {}] or [null, null]
+        return result[1];
+    }
+    async addTaskQueues(keys) {
+        const multi = this.storeClient.multi();
+        const zsetKey = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId });
+        for (const key of keys) {
+            multi.zadd(zsetKey, 'NX', Date.now(), key);
+        }
+        await multi.exec();
+    }
+}
+exports.IORedisStoreService = IORedisStoreService;

package/build/services/store/providers/redis/redis.d.ts

@@ -0,0 +1,18 @@
+import { StoreInitializable } from '../store-initializable';
+import { RedisRedisClientType as RedisClientType, RedisRedisMultiType as RedisMultiType } from '../../../../types/redis';
+import { RedisStoreBase } from './_base';
+declare class RedisStoreService extends RedisStoreBase<RedisClientType, RedisMultiType> implements StoreInitializable {
+    constructor(storeClient: RedisClientType);
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated
+     */
+    transact(): RedisMultiType;
+    exec(...args: any[]): Promise<string | string[] | string[][]>;
+    setnxex(key: string, value: string, expireSeconds: number): Promise<boolean>;
+    zAdd(key: string, score: number | string, value: string | number, redisMulti?: RedisMultiType): Promise<any>;
+    zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null>;
+}
+export { RedisStoreService };

package/build/services/store/providers/redis/redis.js

@@ -0,0 +1,199 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisStoreService = void 0;
+const enums_1 = require("../../../../modules/enums");
+const _base_1 = require("./_base");
+class RedisStoreService extends _base_1.RedisStoreBase {
+    constructor(storeClient) {
+        super(storeClient);
+        this.commands = {
+            get: 'GET',
+            set: 'SET',
+            setnx: 'SETNX',
+            del: 'DEL',
+            expire: 'EXPIRE',
+            hscan: 'HSCAN',
+            hset: 'HSET',
+            hsetnx: 'HSETNX',
+            hincrby: 'HINCRBY',
+            hdel: 'HDEL',
+            hget: 'HGET',
+            hmget: 'HMGET',
+            hgetall: 'HGETALL',
+            hincrbyfloat: 'HINCRBYFLOAT',
+            zrange: 'ZRANGE',
+            zrangebyscore_withscores: 'ZRANGEBYSCORE_WITHSCORES',
+            zrangebyscore: 'ZRANGEBYSCORE',
+            zrem: 'ZREM',
+            zadd: 'ZADD',
+            lmove: 'LMOVE',
+            lrange: 'LRANGE',
+            lpop: 'LPOP',
+            rename: 'RENAME',
+            rpush: 'RPUSH',
+            scan: 'SCAN',
+            xack: 'XACK',
+            xdel: 'XDEL',
+            xlen: 'XLEN',
+        };
+    }
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated
+     */
+    transact() {
+        const my = this;
+        if (enums_1.HMSH_IS_CLUSTER) {
+            const commands = [];
+            const addCommand = (command, args) => {
+                commands.push({ command: command.toUpperCase(), args });
+                return multiInstance;
+            };
+            const multiInstance = {
+                sendCommand(command, ...args) {
+                    return my.storeClient.sendCommand([command, ...args]);
+                },
+                async exec() {
+                    if (commands.length === 0)
+                        return [];
+                    const sameKey = commands.every((cmd) => {
+                        return cmd.args[0] === commands[0].args[0];
+                    });
+                    if (sameKey) {
+                        const multi = my.storeClient.multi();
+                        commands.forEach((cmd) => {
+                            if (cmd.command === 'ZADD') {
+                                return multi.ZADD(cmd.args[0], cmd.args[1], cmd.args[2]);
+                            }
+                            return multi[cmd.command](...cmd.args);
+                        });
+                        const results = await multi.exec();
+                        return results.map((item) => item);
+                    }
+                    else {
+                        return Promise.all(commands.map((cmd) => {
+                            if (cmd.command === 'ZADD') {
+                                return my.storeClient.ZADD(cmd.args[0], cmd.args[1], cmd.args[2]);
+                            }
+                            return my.storeClient[cmd.command](...cmd.args);
+                        }));
+                    }
+                },
+                XADD(key, id, fields, message) {
+                    return addCommand('XADD', [key, id, fields, message]);
+                },
+                XACK(key, group, id) {
+                    return addCommand('XACK', [key, group, id]);
+                },
+                XDEL(key, id) {
+                    return addCommand('XDEL', [key, id]);
+                },
+                XLEN(key) {
+                    return addCommand('XLEN', [key]);
+                },
+                XCLAIM(key, group, consumer, minIdleTime, id, ...args) {
+                    return addCommand('XCLAIM', [
+                        key,
+                        group,
+                        consumer,
+                        minIdleTime,
+                        id,
+                        ...args,
+                    ]);
+                },
+                XPENDING(key, group, start, end, count, consumer) {
+                    return addCommand('XPENDING', [
+                        key,
+                        group,
+                        start,
+                        end,
+                        count,
+                        consumer,
+                    ]);
+                },
+                DEL: function (key) {
+                    return addCommand('DEL', [key]);
+                },
+                EXPIRE: function (key, seconds) {
+                    return addCommand('EXPIRE', [key, seconds]);
+                },
+                HDEL(key, itemId) {
+                    return addCommand('HDEL', [key, itemId]);
+                },
+                HGET(key, itemId) {
+                    return addCommand('HGET', [key, itemId]);
+                },
+                HGETALL(key) {
+                    return addCommand('HGETALL', [key]);
+                },
+                HINCRBYFLOAT(key, itemId, value) {
+                    return addCommand('HINCRBYFLOAT', [key, itemId, value]);
+                },
+                HMGET(key, itemIds) {
+                    return addCommand('HMGET', [key, itemIds]);
+                },
+                HSET(key, values) {
+                    return addCommand('HSET', [key, values]);
+                },
+                LRANGE(key, start, end) {
+                    return addCommand('LRANGE', [key, start, end]);
+                },
+                RPUSH(key, items) {
+                    return addCommand('RPUSH', [key, items]);
+                },
+                ZADD(key, args, opts) {
+                    return addCommand('ZADD', [key, args, opts]);
+                },
+                XGROUP(command, key, groupName, id, mkStream) {
+                    return addCommand('XGROUP', [command, key, groupName, id, mkStream]);
+                },
+                EXISTS: function (key) {
+                    throw new Error('Function not implemented.');
+                },
+                HMPUSH: function (key, values) {
+                    throw new Error('Function not implemented.');
+                },
+                LPUSH: function (key, items) {
+                    throw new Error('Function not implemented.');
+                },
+                SET: function (key, value) {
+                    throw new Error('Function not implemented.');
+                },
+                ZRANGE_WITHSCORES: function (key, start, end) {
+                    throw new Error('Function not implemented.');
+                },
+                ZRANK: function (key, member) {
+                    throw new Error('Function not implemented.');
+                },
+                ZSCORE: function (key, value) {
+                    throw new Error('Function not implemented.');
+                },
+            };
+            return multiInstance;
+        }
+        return this.storeClient.multi();
+    }
+    async exec(...args) {
+        return await this.storeClient.sendCommand(args);
+    }
+    async setnxex(key, value, expireSeconds) {
+        const status = await this.storeClient[this.commands.set](key, value, { NX: true, EX: expireSeconds });
+        return this.isSuccessful(status);
+    }
+    async zAdd(key, score, value, redisMulti) {
+        return await (redisMulti || this.storeClient)[this.commands.zadd](key, {
+            score: score,
+            value: value.toString(),
+        });
+    }
+    async zRangeByScore(key, score, value) {
+        const result = await this.storeClient[this.commands.zrangebyscore](key, score, value);
+        if (result?.length > 0) {
+            return result[0];
+        }
+        return null;
+    }
+}
+exports.RedisStoreService = RedisStoreService;

package/build/services/stream/factory.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamServiceFactory = void 0;
 const utils_1 = require("../../modules/utils");
+const ioredis_1 = require("./providers/redis/ioredis");
+const redis_1 = require("./providers/redis/redis");
 const nats_1 = require("./providers/nats/nats");
 const postgres_1 = require("./providers/postgres/postgres");
 class StreamServiceFactory {
@@ -9,10 +11,24 @@ class StreamServiceFactory {
         let service;
         const providerType = (0, utils_1.identifyProvider)(provider);
         if (providerType === 'nats') {
-            service = new nats_1.NatsStreamService(provider, storeProvider);
+            let redisStoreProvider;
+            if ((0, utils_1.identifyProvider)(storeProvider) === 'redis') {
+                redisStoreProvider = storeProvider;
+            }
+            else {
+                //ioredis
+                redisStoreProvider = storeProvider;
+            }
+            service = new nats_1.NatsStreamService(provider, redisStoreProvider);
         }
         else if (providerType === 'postgres') {
             service = new postgres_1.PostgresStreamService(provider, storeProvider);
+        }
+        else if (providerType === 'redis') {
+            service = new redis_1.RedisStreamService(provider, storeProvider);
+        }
+        else if (providerType === 'ioredis') {
+            service = new ioredis_1.IORedisStreamService(provider, storeProvider);
         } //etc register other providers here
         await service.init(namespace, appId, logger);
         return service;

package/build/services/stream/providers/postgres/kvtables.js

@@ -73,8 +73,13 @@ function hashStringToInt(str) {
     return Math.abs(hash);
 }
 async function checkIfTablesExist(client, schemaName, tableName) {
-    const result = await client.query(`SELECT to_regclass('${tableName}') AS t`);
-    return result.rows[0].t !== null;
+    // Check both streams table exists AND roles table (from store provider)
+    // The roles table is created by the store provider and is used for scout role coordination
+    const result = await client.query(`SELECT 
+       to_regclass($1) AS streams_table,
+       to_regclass($2) AS roles_table`, [tableName, `${schemaName}.roles`]);
+    return result.rows[0].streams_table !== null &&
+        result.rows[0].roles_table !== null;
 }
 async function waitForTablesCreation(streamClient, lockId, schemaName, tableName, logger) {
     let retries = 0;
@@ -124,6 +129,11 @@ async function createTables(client, schemaName, tableName) {
       reserved_at TIMESTAMPTZ,
       reserved_by TEXT,
       expired_at TIMESTAMPTZ,
+      max_retry_attempts INT DEFAULT 3,
+      backoff_coefficient NUMERIC DEFAULT 10,
+      maximum_interval_seconds INT DEFAULT 120,
+      visible_at TIMESTAMPTZ DEFAULT NOW(),
+      retry_attempt INT DEFAULT 0,
       PRIMARY KEY (stream_name, id)
     ) PARTITION BY HASH (stream_name);
   `);
@@ -135,16 +145,16 @@ async function createTables(client, schemaName, tableName) {
       FOR VALUES WITH (modulus 8, remainder ${i});
     `);
     }
-    // Index for active messages
+    // Index for active messages (includes visible_at for visibility timeout support)
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_streams_active_messages 
-    ON ${tableName} (group_name, stream_name, reserved_at, id)
+    ON ${tableName} (group_name, stream_name, reserved_at, visible_at, id)
     WHERE reserved_at IS NULL AND expired_at IS NULL;
   `);
-    // Optimized index for the simplified fetchMessages query
+    // Optimized index for the simplified fetchMessages query (includes visible_at)
     await client.query(`
     CREATE INDEX IF NOT EXISTS idx_streams_message_fetch 
-    ON ${tableName} (stream_name, group_name, id)
+    ON ${tableName} (stream_name, group_name, visible_at, id)
     WHERE expired_at IS NULL;
   `);
     // Index for expired messages
@@ -166,7 +176,7 @@ async function createTables(client, schemaName, tableName) {
     // `);
 }
 async function createNotificationTriggers(client, schemaName, tableName) {
-    // Create the notification function
+    // Create the notification function for INSERT events
     await client.query(`
     CREATE OR REPLACE FUNCTION ${schemaName}.notify_new_stream_message()
     RETURNS TRIGGER AS $$
@@ -174,24 +184,26 @@ async function createNotificationTriggers(client, schemaName, tableName) {
       channel_name TEXT;
       payload JSON;
     BEGIN
-      -- Create channel name: stream_{stream_name}_{group_name}
-      -- Truncate if too long (PostgreSQL channel names limited to 63 chars)
-      channel_name := 'stream_' || NEW.stream_name || '_' || NEW.group_name;
-      IF length(channel_name) > 63 THEN
-        channel_name := left(channel_name, 63);
+      -- Only notify if message is immediately visible
+      -- Messages with visibility timeout will be notified when they become visible
+      IF NEW.visible_at <= NOW() THEN
+        -- Create channel name: stream_{stream_name}_{group_name}
+        -- Truncate if too long (PostgreSQL channel names limited to 63 chars)
+        channel_name := 'stream_' || NEW.stream_name || '_' || NEW.group_name;
+        IF length(channel_name) > 63 THEN
+          channel_name := left(channel_name, 63);
+        END IF;
+        
+        -- Create minimal payload with only required fields
+        payload := json_build_object(
+          'stream_name', NEW.stream_name,
+          'group_name', NEW.group_name
+        );
+        
+        -- Send notification
+        PERFORM pg_notify(channel_name, payload::text);
       END IF;
       
-      -- Create payload with message details
-      payload := json_build_object(
-        'id', NEW.id,
-        'stream_name', NEW.stream_name,
-        'group_name', NEW.group_name,
-        'created_at', extract(epoch from NEW.created_at)
-      );
-      
-      -- Send notification
-      PERFORM pg_notify(channel_name, payload::text);
-      
       RETURN NEW;
     END;
     $$ LANGUAGE plpgsql;
@@ -204,6 +216,47 @@ async function createNotificationTriggers(client, schemaName, tableName) {
       FOR EACH ROW
       EXECUTE FUNCTION ${schemaName}.notify_new_stream_message();
   `);
+    // Create helper function to notify about messages with expired visibility timeouts
+    // This is called periodically by the router scout for responsive retry processing
+    await client.query(`
+    CREATE OR REPLACE FUNCTION ${schemaName}.notify_visible_messages()
+    RETURNS INTEGER AS $$
+    DECLARE
+      msg RECORD;
+      channel_name TEXT;
+      payload JSON;
+      notification_count INTEGER := 0;
+    BEGIN
+      -- Find all distinct streams with messages that are now visible
+      -- Router will drain all messages when notified, so we just notify each channel once
+      FOR msg IN 
+        SELECT DISTINCT stream_name, group_name
+        FROM ${tableName}
+        WHERE visible_at <= NOW()
+          AND reserved_at IS NULL
+          AND expired_at IS NULL
+        LIMIT 100  -- Prevent overwhelming the system
+      LOOP
+        -- Create channel name (same logic as INSERT trigger)
+        channel_name := 'stream_' || msg.stream_name || '_' || msg.group_name;
+        IF length(channel_name) > 63 THEN
+          channel_name := left(channel_name, 63);
+        END IF;
+        
+        -- Send minimal notification with only required fields
+        payload := json_build_object(
+          'stream_name', msg.stream_name,
+          'group_name', msg.group_name
+        );
+        
+        PERFORM pg_notify(channel_name, payload::text);
+        notification_count := notification_count + 1;
+      END LOOP;
+      
+      RETURN notification_count;
+    END;
+    $$ LANGUAGE plpgsql;
+  `);
 }
 function getNotificationChannelName(streamName, groupName) {
     const channelName = `stream_${streamName}_${groupName}`;

package/build/services/stream/providers/postgres/lifecycle.d.ts

@@ -0,0 +1,19 @@
+import { ILogger } from '../../../logger';
+import { PostgresClientType } from '../../../../types/postgres';
+import { ProviderClient } from '../../../../types/provider';
+/**
+ * Create a stream (no-op for PostgreSQL - streams are created implicitly).
+ */
+export declare function createStream(streamName: string): Promise<boolean>;
+/**
+ * Delete a stream or all streams.
+ */
+export declare function deleteStream(client: PostgresClientType & ProviderClient, tableName: string, streamName: string, logger: ILogger): Promise<boolean>;
+/**
+ * Create a consumer group (no-op for PostgreSQL - groups are created implicitly).
+ */
+export declare function createConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+/**
+ * Delete a consumer group (removes all messages for that group).
+ */
+export declare function deleteConsumerGroup(client: PostgresClientType & ProviderClient, tableName: string, streamName: string, groupName: string, logger: ILogger): Promise<boolean>;