@hotmeshio/hotmesh 0.4.0 → 0.4.2

package/build/services/store/providers/redis/ioredis.d.ts

@@ -0,0 +1,20 @@
+import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../../types/redis';
+import { StoreInitializable } from '../store-initializable';
+import { RedisStoreBase } from './_base';
+declare class IORedisStoreService extends RedisStoreBase<RedisClientType, RedisMultiType> implements StoreInitializable {
+    constructor(storeClient: RedisClientType);
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated. this is effectiely a wrapper for
+     * `multi` but is closer to `pipeline` in terms of usage when
+     * promises are used.
+     */
+    transact(): RedisMultiType;
+    exec(...args: any[]): Promise<string | string[] | string[][]>;
+    setnxex(key: string, value: string, expireSeconds: number): Promise<boolean>;
+    hGetAllResult(result: any): any;
+    addTaskQueues(keys: string[]): Promise<void>;
+}
+export { IORedisStoreService };

package/build/services/store/providers/redis/ioredis.js

@@ -0,0 +1,180 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisStoreService = void 0;
+const key_1 = require("../../../../modules/key");
+const enums_1 = require("../../../../modules/enums");
+const _base_1 = require("./_base");
+class IORedisStoreService extends _base_1.RedisStoreBase {
+    constructor(storeClient) {
+        super(storeClient);
+        this.commands = {
+            get: 'get',
+            set: 'set',
+            setnx: 'setnx',
+            del: 'del',
+            expire: 'expire',
+            hset: 'hset',
+            hscan: 'hscan',
+            hsetnx: 'hsetnx',
+            hincrby: 'hincrby',
+            hdel: 'hdel',
+            hget: 'hget',
+            hmget: 'hmget',
+            hgetall: 'hgetall',
+            hincrbyfloat: 'hincrbyfloat',
+            zrank: 'zrank',
+            zrange: 'zrange',
+            zrangebyscore_withscores: 'zrangebyscore',
+            zrangebyscore: 'zrangebyscore',
+            zrem: 'zrem',
+            zadd: 'zadd',
+            lmove: 'lmove',
+            lpop: 'lpop',
+            lrange: 'lrange',
+            rename: 'rename',
+            rpush: 'rpush',
+            scan: 'scan',
+            xack: 'xack',
+            xdel: 'xdel',
+        };
+    }
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated. this is effectiely a wrapper for
+     * `multi` but is closer to `pipeline` in terms of usage when
+     * promises are used.
+     */
+    transact() {
+        const my = this;
+        if (enums_1.HMSH_IS_CLUSTER) {
+            const commands = [];
+            const addCommand = (command, args) => {
+                commands.push({ command, args });
+                return multiInstance;
+            };
+            const multiInstance = {
+                sendCommand(command) {
+                    return my.storeClient.sendCommand(command);
+                },
+                async exec() {
+                    if (commands.length === 0)
+                        return [];
+                    const sameKey = commands.every((cmd) => {
+                        return cmd.args[0] === commands[0].args[0];
+                    });
+                    if (sameKey) {
+                        const multi = my.storeClient.multi();
+                        commands.forEach((cmd) => multi[cmd.command](...cmd.args));
+                        const results = await multi.exec();
+                        return results.map((item) => item);
+                    }
+                    else {
+                        return Promise.all(commands.map((cmd) => my.storeClient[cmd.command](...cmd.args)));
+                    }
+                },
+                xadd(key, id, fields, message) {
+                    return addCommand('xadd', [key, id, fields, message]);
+                },
+                xack(key, group, id) {
+                    return addCommand('xack', [key, group, id]);
+                },
+                xdel(key, id) {
+                    return addCommand('xdel', [key, id]);
+                },
+                xlen(key) {
+                    return addCommand('xlen', [key]);
+                },
+                xpending(key, group, start, end, count, consumer) {
+                    return addCommand('xpending', [
+                        key,
+                        group,
+                        start,
+                        end,
+                        count,
+                        consumer,
+                    ]);
+                },
+                xclaim(key, group, consumer, minIdleTime, id, ...args) {
+                    return addCommand('xclaim', [
+                        key,
+                        group,
+                        consumer,
+                        minIdleTime,
+                        id,
+                        ...args,
+                    ]);
+                },
+                del(key) {
+                    return addCommand('del', [key]);
+                },
+                expire: function (key, seconds) {
+                    return addCommand('expire', [key, seconds]);
+                },
+                hdel(key, itemId) {
+                    return addCommand('hdel', [key, itemId]);
+                },
+                hget(key, itemId) {
+                    return addCommand('hget', [key, itemId]);
+                },
+                hgetall(key) {
+                    return addCommand('hgetall', [key]);
+                },
+                hincrbyfloat(key, itemId, value) {
+                    return addCommand('hincrbyfloat', [key, itemId, value]);
+                },
+                hmget(key, itemIds) {
+                    return addCommand('hmget', [key, itemIds]);
+                },
+                hset(key, values) {
+                    return addCommand('hset', [key, values]);
+                },
+                lrange(key, start, end) {
+                    return addCommand('lrange', [key, start, end]);
+                },
+                rpush(key, value) {
+                    return addCommand('rpush', [key, value]);
+                },
+                zadd(...args) {
+                    return addCommand('zadd', args);
+                },
+                xgroup(command, key, groupName, id, mkStream) {
+                    return addCommand('xgroup', [command, key, groupName, id, mkStream]);
+                },
+            };
+            return multiInstance;
+        }
+        return this.storeClient.multi();
+    }
+    async exec(...args) {
+        const response = await this.storeClient.call.apply(this.storeClient, args);
+        if (typeof response === 'string') {
+            return response;
+        }
+        else if (Array.isArray(response)) {
+            if (Array.isArray(response[0])) {
+                return response;
+            }
+            return response;
+        }
+        return response;
+    }
+    async setnxex(key, value, expireSeconds) {
+        const status = await this.storeClient[this.commands.set](key, value, 'NX', 'EX', expireSeconds.toString());
+        return this.isSuccessful(status);
+    }
+    hGetAllResult(result) {
+        //ioredis response signature is [null, {}] or [null, null]
+        return result[1];
+    }
+    async addTaskQueues(keys) {
+        const multi = this.storeClient.multi();
+        const zsetKey = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId });
+        for (const key of keys) {
+            multi.zadd(zsetKey, 'NX', Date.now(), key);
+        }
+        await multi.exec();
+    }
+}
+exports.IORedisStoreService = IORedisStoreService;

package/build/services/store/providers/redis/redis.d.ts

@@ -0,0 +1,18 @@
+import { StoreInitializable } from '../store-initializable';
+import { RedisRedisClientType as RedisClientType, RedisRedisMultiType as RedisMultiType } from '../../../../types/redis';
+import { RedisStoreBase } from './_base';
+declare class RedisStoreService extends RedisStoreBase<RedisClientType, RedisMultiType> implements StoreInitializable {
+    constructor(storeClient: RedisClientType);
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated
+     */
+    transact(): RedisMultiType;
+    exec(...args: any[]): Promise<string | string[] | string[][]>;
+    setnxex(key: string, value: string, expireSeconds: number): Promise<boolean>;
+    zAdd(key: string, score: number | string, value: string | number, redisMulti?: RedisMultiType): Promise<any>;
+    zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null>;
+}
+export { RedisStoreService };

package/build/services/store/providers/redis/redis.js

@@ -0,0 +1,199 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisStoreService = void 0;
+const enums_1 = require("../../../../modules/enums");
+const _base_1 = require("./_base");
+class RedisStoreService extends _base_1.RedisStoreBase {
+    constructor(storeClient) {
+        super(storeClient);
+        this.commands = {
+            get: 'GET',
+            set: 'SET',
+            setnx: 'SETNX',
+            del: 'DEL',
+            expire: 'EXPIRE',
+            hscan: 'HSCAN',
+            hset: 'HSET',
+            hsetnx: 'HSETNX',
+            hincrby: 'HINCRBY',
+            hdel: 'HDEL',
+            hget: 'HGET',
+            hmget: 'HMGET',
+            hgetall: 'HGETALL',
+            hincrbyfloat: 'HINCRBYFLOAT',
+            zrange: 'ZRANGE',
+            zrangebyscore_withscores: 'ZRANGEBYSCORE_WITHSCORES',
+            zrangebyscore: 'ZRANGEBYSCORE',
+            zrem: 'ZREM',
+            zadd: 'ZADD',
+            lmove: 'LMOVE',
+            lrange: 'LRANGE',
+            lpop: 'LPOP',
+            rename: 'RENAME',
+            rpush: 'RPUSH',
+            scan: 'SCAN',
+            xack: 'XACK',
+            xdel: 'XDEL',
+            xlen: 'XLEN',
+        };
+    }
+    /**
+     * When in cluster mode, the transact wrapper only
+     * sends commands to the same node/shard if they share a key.
+     * All other commands are sent simultaneouslyusing Promise.all
+     * and are then collated
+     */
+    transact() {
+        const my = this;
+        if (enums_1.HMSH_IS_CLUSTER) {
+            const commands = [];
+            const addCommand = (command, args) => {
+                commands.push({ command: command.toUpperCase(), args });
+                return multiInstance;
+            };
+            const multiInstance = {
+                sendCommand(command, ...args) {
+                    return my.storeClient.sendCommand([command, ...args]);
+                },
+                async exec() {
+                    if (commands.length === 0)
+                        return [];
+                    const sameKey = commands.every((cmd) => {
+                        return cmd.args[0] === commands[0].args[0];
+                    });
+                    if (sameKey) {
+                        const multi = my.storeClient.multi();
+                        commands.forEach((cmd) => {
+                            if (cmd.command === 'ZADD') {
+                                return multi.ZADD(cmd.args[0], cmd.args[1], cmd.args[2]);
+                            }
+                            return multi[cmd.command](...cmd.args);
+                        });
+                        const results = await multi.exec();
+                        return results.map((item) => item);
+                    }
+                    else {
+                        return Promise.all(commands.map((cmd) => {
+                            if (cmd.command === 'ZADD') {
+                                return my.storeClient.ZADD(cmd.args[0], cmd.args[1], cmd.args[2]);
+                            }
+                            return my.storeClient[cmd.command](...cmd.args);
+                        }));
+                    }
+                },
+                XADD(key, id, fields, message) {
+                    return addCommand('XADD', [key, id, fields, message]);
+                },
+                XACK(key, group, id) {
+                    return addCommand('XACK', [key, group, id]);
+                },
+                XDEL(key, id) {
+                    return addCommand('XDEL', [key, id]);
+                },
+                XLEN(key) {
+                    return addCommand('XLEN', [key]);
+                },
+                XCLAIM(key, group, consumer, minIdleTime, id, ...args) {
+                    return addCommand('XCLAIM', [
+                        key,
+                        group,
+                        consumer,
+                        minIdleTime,
+                        id,
+                        ...args,
+                    ]);
+                },
+                XPENDING(key, group, start, end, count, consumer) {
+                    return addCommand('XPENDING', [
+                        key,
+                        group,
+                        start,
+                        end,
+                        count,
+                        consumer,
+                    ]);
+                },
+                DEL: function (key) {
+                    return addCommand('DEL', [key]);
+                },
+                EXPIRE: function (key, seconds) {
+                    return addCommand('EXPIRE', [key, seconds]);
+                },
+                HDEL(key, itemId) {
+                    return addCommand('HDEL', [key, itemId]);
+                },
+                HGET(key, itemId) {
+                    return addCommand('HGET', [key, itemId]);
+                },
+                HGETALL(key) {
+                    return addCommand('HGETALL', [key]);
+                },
+                HINCRBYFLOAT(key, itemId, value) {
+                    return addCommand('HINCRBYFLOAT', [key, itemId, value]);
+                },
+                HMGET(key, itemIds) {
+                    return addCommand('HMGET', [key, itemIds]);
+                },
+                HSET(key, values) {
+                    return addCommand('HSET', [key, values]);
+                },
+                LRANGE(key, start, end) {
+                    return addCommand('LRANGE', [key, start, end]);
+                },
+                RPUSH(key, items) {
+                    return addCommand('RPUSH', [key, items]);
+                },
+                ZADD(key, args, opts) {
+                    return addCommand('ZADD', [key, args, opts]);
+                },
+                XGROUP(command, key, groupName, id, mkStream) {
+                    return addCommand('XGROUP', [command, key, groupName, id, mkStream]);
+                },
+                EXISTS: function (key) {
+                    throw new Error('Function not implemented.');
+                },
+                HMPUSH: function (key, values) {
+                    throw new Error('Function not implemented.');
+                },
+                LPUSH: function (key, items) {
+                    throw new Error('Function not implemented.');
+                },
+                SET: function (key, value) {
+                    throw new Error('Function not implemented.');
+                },
+                ZRANGE_WITHSCORES: function (key, start, end) {
+                    throw new Error('Function not implemented.');
+                },
+                ZRANK: function (key, member) {
+                    throw new Error('Function not implemented.');
+                },
+                ZSCORE: function (key, value) {
+                    throw new Error('Function not implemented.');
+                },
+            };
+            return multiInstance;
+        }
+        return this.storeClient.multi();
+    }
+    async exec(...args) {
+        return await this.storeClient.sendCommand(args);
+    }
+    async setnxex(key, value, expireSeconds) {
+        const status = await this.storeClient[this.commands.set](key, value, { NX: true, EX: expireSeconds });
+        return this.isSuccessful(status);
+    }
+    async zAdd(key, score, value, redisMulti) {
+        return await (redisMulti || this.storeClient)[this.commands.zadd](key, {
+            score: score,
+            value: value.toString(),
+        });
+    }
+    async zRangeByScore(key, score, value) {
+        const result = await this.storeClient[this.commands.zrangebyscore](key, score, value);
+        if (result?.length > 0) {
+            return result[0];
+        }
+        return null;
+    }
+}
+exports.RedisStoreService = RedisStoreService;

package/build/services/store/providers/store-initializable.d.ts

@@ -0,0 +1,5 @@
+import { ILogger } from '../../logger';
+import { HotMeshApps } from '../../../types/hotmesh';
+export interface StoreInitializable {
+    init(namespace: string, appId: string, logger: ILogger): Promise<HotMeshApps>;
+}

package/build/services/store/providers/store-initializable.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/build/services/stream/factory.d.ts

@@ -0,0 +1,8 @@
+import { ILogger } from '../logger';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { StreamInitializable } from './providers/stream-initializable';
+import { StreamService } from './index';
+declare class StreamServiceFactory {
+    static init(provider: ProviderClient, storeProvider: ProviderClient, namespace: string, appId: string, logger: ILogger): Promise<StreamService<ProviderClient, ProviderTransaction> & StreamInitializable>;
+}
+export { StreamServiceFactory };

package/build/services/stream/factory.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const ioredis_1 = require("./providers/redis/ioredis");
+const redis_1 = require("./providers/redis/redis");
+const nats_1 = require("./providers/nats/nats");
+const postgres_1 = require("./providers/postgres/postgres");
+class StreamServiceFactory {
+    static async init(provider, storeProvider, namespace, appId, logger) {
+        let service;
+        const providerType = (0, utils_1.identifyProvider)(provider);
+        if (providerType === 'nats') {
+            let storeProvider;
+            if ((0, utils_1.identifyProvider)(storeProvider) === 'redis') {
+                storeProvider = storeProvider;
+            }
+            else {
+                //ioredis
+                storeProvider = storeProvider;
+            }
+            service = new nats_1.NatsStreamService(provider, storeProvider);
+        }
+        else if (providerType === 'postgres') {
+            service = new postgres_1.PostgresStreamService(provider, storeProvider);
+        }
+        else if (providerType === 'redis') {
+            service = new redis_1.RedisStreamService(provider, storeProvider);
+        }
+        else if (providerType === 'ioredis') {
+            service = new ioredis_1.IORedisStreamService(provider, storeProvider);
+        } //etc register other providers here
+        await service.init(namespace, appId, logger);
+        return service;
+    }
+}
+exports.StreamServiceFactory = StreamServiceFactory;

package/build/services/stream/index.d.ts

@@ -0,0 +1,69 @@
+import { ILogger } from '../logger';
+import { PublishMessageConfig, StreamConfig, StreamMessage, StreamStats } from '../../types/stream';
+import { StringAnyType } from '../../types';
+import { KeyStoreParams, KeyType } from '../../modules/key';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+export declare abstract class StreamService<ClientProvider extends ProviderClient, TransactionProvider extends ProviderTransaction> {
+    protected streamClient: ClientProvider;
+    protected storeClient: ProviderClient;
+    protected namespace: string;
+    protected logger: ILogger;
+    protected appId: string;
+    protected config: StreamConfig;
+    constructor(streamClient: ClientProvider, storeClient: ProviderClient, config?: StreamConfig);
+    abstract init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    abstract mintKey(type: KeyType, params: KeyStoreParams): string;
+    abstract createStream(streamName: string): Promise<boolean>;
+    abstract deleteStream(streamName: string): Promise<boolean>;
+    abstract createConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    abstract deleteConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    abstract publishMessages(streamName: string, messages: string[], options?: PublishMessageConfig): Promise<string[] | ProviderTransaction>;
+    abstract consumeMessages(streamName: string, groupName: string, consumerName: string, options?: {
+        batchSize?: number;
+        blockTimeout?: number;
+        autoAck?: boolean;
+        enableBackoff?: boolean;
+        initialBackoff?: number;
+        maxBackoff?: number;
+        maxRetries?: number;
+        enableNotifications?: boolean;
+        notificationCallback?: (messages: StreamMessage[]) => void;
+    }): Promise<StreamMessage[]>;
+    abstract transact(): ProviderTransaction;
+    abstract ackAndDelete(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
+    abstract acknowledgeMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number | TransactionProvider>;
+    abstract deleteMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number | TransactionProvider>;
+    abstract retryMessages(streamName: string, groupName: string, options?: {
+        consumerName?: string;
+        minIdleTime?: number;
+        messageIds?: string[];
+        delay?: number;
+        maxRetries?: number;
+        limit?: number;
+    }): Promise<StreamMessage[]>;
+    abstract getStreamStats(streamName: string): Promise<StreamStats>;
+    abstract getStreamDepth(streamName: string): Promise<number>;
+    abstract getStreamDepths(streamName: {
+        stream: string;
+    }[]): Promise<{
+        stream: string;
+        depth: number;
+    }[]>;
+    abstract trimStream(streamName: string, options: {
+        maxLen?: number;
+        maxAge?: number;
+        exactLimit?: boolean;
+    }): Promise<number>;
+    abstract getProviderSpecificFeatures(): {
+        supportsBatching: boolean;
+        supportsDeadLetterQueue: boolean;
+        supportsOrdering: boolean;
+        supportsTrimming: boolean;
+        supportsRetry: boolean;
+        supportsNotifications?: boolean;
+        maxMessageSize: number;
+        maxBatchSize: number;
+    };
+    stopNotificationConsumer?(streamName: string, groupName: string): Promise<void>;
+    cleanup?(): Promise<void>;
+}

package/build/services/stream/index.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamService = void 0;
+class StreamService {
+    constructor(streamClient, storeClient, config = {}) {
+        this.streamClient = streamClient;
+        this.storeClient = storeClient;
+        this.config = config;
+    }
+}
+exports.StreamService = StreamService;

package/build/services/stream/providers/nats/nats.d.ts

@@ -0,0 +1,60 @@
+import { ILogger } from '../../../logger';
+import { StreamService } from '../../index';
+import { PublishMessageConfig, StreamConfig, StreamMessage, StreamStats } from '../../../../types/stream';
+import { KeyStoreParams, StringAnyType } from '../../../../types';
+import { NatsJetStreamType, NatsJetStreamManager as NatsJetStreamManagerType, NatsPubAckType, NatsClientType } from '../../../../types/nats';
+import { KeyType } from '../../../../modules/key';
+import { ProviderClient, ProviderTransaction } from '../../../../types/provider';
+declare class NatsStreamService extends StreamService<NatsClientType, NatsPubAckType> {
+    jetstream: NatsJetStreamType;
+    jsm: NatsJetStreamManagerType;
+    constructor(streamClient: NatsClientType, storeClient: ProviderClient, config?: StreamConfig);
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    transact(): ProviderTransaction;
+    createStream(streamName: string): Promise<boolean>;
+    deleteStream(streamName: string): Promise<boolean>;
+    createConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    deleteConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    publishMessages(streamName: string, messages: string[], options?: PublishMessageConfig): Promise<string[] | ProviderTransaction>;
+    consumeMessages(streamName: string, groupName: string, consumerName: string, options?: {
+        batchSize?: number;
+        blockTimeout?: number;
+        autoAck?: boolean;
+    }): Promise<StreamMessage[]>;
+    ackAndDelete(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
+    acknowledgeMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
+    deleteMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
+    retryMessages(streamName: string, groupName: string, options?: {
+        consumerName?: string;
+        minIdleTime?: number;
+        messageIds?: string[];
+        delay?: number;
+        maxRetries?: number;
+        limit?: number;
+    }): Promise<StreamMessage[]>;
+    getStreamStats(streamName: string): Promise<StreamStats>;
+    getStreamDepth(streamName: string): Promise<number>;
+    getStreamDepths(streamNames: {
+        stream: string;
+    }[]): Promise<{
+        stream: string;
+        depth: number;
+    }[]>;
+    trimStream(streamName: string, options: {
+        maxLen?: number;
+        maxAge?: number;
+        exactLimit?: boolean;
+    }): Promise<number>;
+    getProviderSpecificFeatures(): {
+        supportsBatching: boolean;
+        supportsDeadLetterQueue: boolean;
+        supportsOrdering: boolean;
+        supportsTrimming: boolean;
+        supportsRetry: boolean;
+        supportsNotifications: boolean;
+        maxMessageSize: number;
+        maxBatchSize: number;
+    };
+}
+export { NatsStreamService };