@hotmeshio/hotmesh 0.5.8 → 0.6.1

package/build/services/sub/providers/postgres/postgres.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgresSubService = void 0;
 const crypto_1 = __importDefault(require("crypto"));
+const enums_1 = require("../../../../modules/enums");
 const key_1 = require("../../../../modules/key");
 const index_1 = require("../../index");
 class PostgresSubService extends index_1.SubService {
@@ -180,8 +181,34 @@ class PostgresSubService extends index_1.SubService {
             appId,
             engineId: topic,
         });
+        let messageToPublish = message;
+        let payload = JSON.stringify(message);
+        // PostgreSQL NOTIFY has a payload limit. If job message exceeds limit,
+        // send a reference message instead - subscriber will fetch via getState.
+        if (payload.length > enums_1.HMSH_NOTIFY_PAYLOAD_LIMIT &&
+            message.type === 'job' &&
+            message.job?.metadata) {
+            const { jid, tpc, app, js } = message.job.metadata;
+            messageToPublish = {
+                type: 'job',
+                topic: message.topic,
+                job: {
+                    metadata: { jid, tpc, app, js },
+                    data: null,
+                },
+                _ref: true,
+            };
+            payload = JSON.stringify(messageToPublish);
+            this.logger.debug('postgres-publish-ref', {
+                originalKey,
+                safeKey,
+                originalSize: JSON.stringify(message).length,
+                refSize: payload.length,
+                jid,
+            });
+        }
         // Publish the message using the safe topic
-        const payload = JSON.stringify(message).replace(/'/g, "''");
+        payload = payload.replace(/'/g, "''");
         await this.storeClient.query(`NOTIFY "${safeKey}", '${payload}'`);
         this.logger.debug(`postgres-publish`, { originalKey, safeKey });
         return true;

package/build/services/sub/providers/redis/ioredis.d.ts

@@ -0,0 +1,20 @@
+import { KeyStoreParams, KeyType } from '../../../../modules/key';
+import { ILogger } from '../../../logger';
+import { SubService } from '../../index';
+import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../../types/redis';
+import { SubscriptionCallback } from '../../../../types/quorum';
+declare class IORedisSubService extends SubService<RedisClientType> {
+    private subscriptions;
+    private messageHandlerSet;
+    private pmessageHandlerSet;
+    constructor(eventClient: RedisClientType, storeClient: RedisClientType);
+    init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    transact(): RedisMultiType;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean>;
+}
+export { IORedisSubService };

package/build/services/sub/providers/redis/ioredis.js

@@ -0,0 +1,150 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisSubService = void 0;
+const key_1 = require("../../../../modules/key");
+const index_1 = require("../../index");
+class IORedisSubService extends index_1.SubService {
+    constructor(eventClient, storeClient) {
+        super(eventClient, storeClient);
+        this.subscriptions = new Map();
+        this.messageHandlerSet = false;
+        this.pmessageHandlerSet = false;
+    }
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.engineId = engineId;
+    }
+    transact() {
+        return this.eventClient.multi();
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, params);
+    }
+    async subscribe(keyType, callback, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        // Add callback to set (supports multiple subscribers per topic)
+        let callbacks = this.subscriptions.get(topic);
+        const isFirstSubscription = !callbacks;
+        if (!callbacks) {
+            callbacks = new Set();
+            this.subscriptions.set(topic, callbacks);
+        }
+        callbacks.add(callback);
+        this.logger.debug('ioredis-subscribe', {
+            topic,
+            isFirstSubscription,
+            totalCallbacks: callbacks.size,
+            messageHandlerSet: this.messageHandlerSet,
+        });
+        // Set up message handler only once
+        if (!this.messageHandlerSet) {
+            this.eventClient.on('message', (channel, message) => {
+                try {
+                    const payload = JSON.parse(message);
+                    const cbs = this.subscriptions.get(channel);
+                    this.logger.debug('ioredis-message-received', {
+                        channel,
+                        callbackCount: cbs?.size || 0,
+                        payload,
+                    });
+                    if (cbs) {
+                        // Call all callbacks for this channel
+                        cbs.forEach(cb => {
+                            try {
+                                cb(channel, payload);
+                            }
+                            catch (err) {
+                                this.logger.error(`Error in callback for ${channel}`, err);
+                            }
+                        });
+                    }
+                }
+                catch (e) {
+                    this.logger.error(`Error parsing message: ${message}`, e);
+                }
+            });
+            this.messageHandlerSet = true;
+        }
+        await this.eventClient.subscribe(topic, (err) => {
+            if (err) {
+                this.logger.error(`Error subscribing to: ${topic}`, err);
+            }
+        });
+    }
+    async unsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        const callbacks = this.subscriptions.get(topic);
+        this.logger.debug('ioredis-unsubscribe', {
+            topic,
+            hadCallbacks: !!callbacks,
+            callbackCount: callbacks?.size || 0,
+        });
+        // Only unsubscribe from Redis if no more callbacks exist
+        if (callbacks && callbacks.size > 0) {
+            this.subscriptions.delete(topic);
+            await this.eventClient.unsubscribe(topic);
+        }
+    }
+    async psubscribe(keyType, callback, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        // Add callback to set (supports multiple subscribers per topic)
+        let callbacks = this.subscriptions.get(topic);
+        if (!callbacks) {
+            callbacks = new Set();
+            this.subscriptions.set(topic, callbacks);
+        }
+        callbacks.add(callback);
+        // Set up pmessage handler only once
+        if (!this.pmessageHandlerSet) {
+            this.eventClient.on('pmessage', (pattern, channel, message) => {
+                try {
+                    const payload = JSON.parse(message);
+                    const cbs = this.subscriptions.get(pattern);
+                    if (cbs) {
+                        // Call all callbacks for this pattern
+                        cbs.forEach(cb => {
+                            try {
+                                cb(channel, payload);
+                            }
+                            catch (err) {
+                                this.logger.error(`Error in callback for ${pattern}`, err);
+                            }
+                        });
+                    }
+                }
+                catch (e) {
+                    this.logger.error(`Error parsing message: ${message}`, e);
+                }
+            });
+            this.pmessageHandlerSet = true;
+        }
+        await this.eventClient.psubscribe(topic, (err) => {
+            if (err) {
+                this.logger.error(`Error subscribing to: ${topic}`, err);
+            }
+        });
+    }
+    async punsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        const callbacks = this.subscriptions.get(topic);
+        // Only unsubscribe from Redis if no more callbacks exist
+        if (callbacks && callbacks.size > 0) {
+            this.subscriptions.delete(topic);
+            await this.eventClient.punsubscribe(topic);
+        }
+    }
+    async publish(keyType, message, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        //NOTE: `storeClient.publish` is used,
+        //      because a Redis connection with subscriptions
+        //      may not publish (is read only).
+        this.logger.debug('ioredis-publish', { topic, message });
+        const status = await this.storeClient.publish(topic, JSON.stringify(message));
+        return status === 1;
+    }
+}
+exports.IORedisSubService = IORedisSubService;

package/build/services/sub/providers/redis/redis.d.ts

@@ -0,0 +1,18 @@
+import { KeyStoreParams, KeyType } from '../../../../modules/key';
+import { ILogger } from '../../../logger';
+import { SubService } from '../../index';
+import { RedisRedisClientType as ClientProvider, RedisRedisMultiType as TransactionProvider } from '../../../../types/redis';
+import { SubscriptionCallback } from '../../../../types/quorum';
+declare class RedisSubService extends SubService<ClientProvider> {
+    private subscriptions;
+    constructor(eventClient: ClientProvider, storeClient: ClientProvider);
+    init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    transact(): TransactionProvider;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean>;
+}
+export { RedisSubService };

package/build/services/sub/providers/redis/redis.js

@@ -0,0 +1,137 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisSubService = void 0;
+const key_1 = require("../../../../modules/key");
+const index_1 = require("../../index");
+class RedisSubService extends index_1.SubService {
+    constructor(eventClient, storeClient) {
+        super(eventClient, storeClient);
+        this.subscriptions = new Map();
+    }
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.engineId = engineId;
+    }
+    transact() {
+        const multi = this.eventClient.multi();
+        return multi;
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, params);
+    }
+    async subscribe(keyType, callback, appId, engineId) {
+        if (this.eventClient) {
+            const topic = this.mintKey(keyType, { appId, engineId });
+            // Add callback to set (supports multiple subscribers per topic)
+            let callbacks = this.subscriptions.get(topic);
+            const isFirstSubscription = !callbacks;
+            if (!callbacks) {
+                callbacks = new Set();
+                this.subscriptions.set(topic, callbacks);
+            }
+            callbacks.add(callback);
+            this.logger.debug('redis-subscribe', {
+                topic,
+                isFirstSubscription,
+                totalCallbacks: callbacks.size,
+            });
+            // Only subscribe to Redis once per topic
+            if (isFirstSubscription) {
+                await this.eventClient.subscribe(topic, (message) => {
+                    try {
+                        const payload = JSON.parse(message);
+                        const cbs = this.subscriptions.get(topic);
+                        this.logger.debug('redis-message-received', {
+                            topic,
+                            callbackCount: cbs?.size || 0,
+                            payload,
+                        });
+                        if (cbs) {
+                            // Call all callbacks for this topic
+                            cbs.forEach(cb => {
+                                try {
+                                    cb(topic, payload);
+                                }
+                                catch (err) {
+                                    this.logger.error(`Error in callback for ${topic}`, err);
+                                }
+                            });
+                        }
+                    }
+                    catch (e) {
+                        this.logger.error(`Error parsing message: ${message}`, e);
+                    }
+                });
+            }
+        }
+    }
+    async unsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        const callbacks = this.subscriptions.get(topic);
+        this.logger.debug('redis-unsubscribe', {
+            topic,
+            hadCallbacks: !!callbacks,
+            callbackCount: callbacks?.size || 0,
+        });
+        // Only unsubscribe from Redis if no more callbacks exist
+        if (callbacks && callbacks.size > 0) {
+            this.subscriptions.delete(topic);
+            await this.eventClient.unsubscribe(topic);
+        }
+    }
+    async psubscribe(keyType, callback, appId, engineId) {
+        if (this.eventClient) {
+            const topic = this.mintKey(keyType, { appId, engineId });
+            // Add callback to set (supports multiple subscribers per topic)
+            let callbacks = this.subscriptions.get(topic);
+            if (!callbacks) {
+                callbacks = new Set();
+                this.subscriptions.set(topic, callbacks);
+            }
+            callbacks.add(callback);
+            await this.eventClient.pSubscribe(topic, (message, channel) => {
+                try {
+                    const payload = JSON.parse(message);
+                    const cbs = this.subscriptions.get(topic);
+                    if (cbs) {
+                        // Call all callbacks for this topic
+                        cbs.forEach(cb => {
+                            try {
+                                cb(channel, payload);
+                            }
+                            catch (err) {
+                                this.logger.error(`Error in callback for ${topic}`, err);
+                            }
+                        });
+                    }
+                }
+                catch (e) {
+                    this.logger.error(`Error parsing message: ${message}`, e);
+                }
+            });
+        }
+    }
+    async punsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        const callbacks = this.subscriptions.get(topic);
+        // Only unsubscribe from Redis if no more callbacks exist
+        if (callbacks && callbacks.size > 0) {
+            this.subscriptions.delete(topic);
+            await this.eventClient.pUnsubscribe(topic);
+        }
+    }
+    async publish(keyType, message, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        //NOTE: `storeClient.publish` is used,
+        //      because a Redis connection with subscriptions
+        //      may not publish (is read only).
+        this.logger.debug('redis-publish', { topic, message });
+        const status = await this.storeClient.publish(topic, JSON.stringify(message));
+        return status > 0;
+    }
+}
+exports.RedisSubService = RedisSubService;

package/build/types/index.d.ts

@@ -17,6 +17,7 @@ export { MeshCallConnectParams, MeshCallExecParams, MeshCallCronParams, MeshCall
 export { PostgresClassType, PostgresClientOptions, PostgresClientType, PostgresConsumerGroup, PostgresPendingMessage, PostgresPoolClientType, PostgresQueryConfigType, PostgresQueryResultType, PostgresStreamMessage, PostgresStreamOptions, PostgresTransaction, } from './postgres';
 export { ActivateMessage, CronMessage, JobMessage, JobMessageCallback, PingMessage, PongMessage, QuorumMessage, QuorumMessageCallback, QuorumProfile, RollCallMessage, RollCallOptions, SubscriptionCallback, SubscriptionOptions, SystemHealth, ThrottleMessage, ThrottleOptions, WorkMessage, } from './quorum';
 export { NatsAckPolicy, NatsAckPolicyExplicitType, NatsClassType, NatsClientType, NatsClientOptions, NatsConsumerConfigType, NatsJetStreamManager, NatsConnection, NatsJetStreamType, NatsConnectionOptions, NatsConsumerConfig, NatsConsumerInfo, NatsConsumerManager, NatsDeliveryInfo, NatsJetStreamOptions, NatsError, NatsErrorType, NatsJetStreamClient, NatsJsMsg, NatsMessageType, NatsMsgExpect, NatsPubAck, NatsPubAckType, NatsPublishOptions, NatsRetentionPolicy, NatsRetentionPolicyWorkqueueType, NatsSequenceInfo, NatsStorageMemoryType, NatsStorageType, NatsStreamConfig, NatsStreamInfo, NatsStreamManager, NatsStreamConfigType, NatsStreamInfoType, NatsStreamOptions, NatsStreamState, NatsTransaction, } from './nats';
+export { RedisClass, RedisRedisClientType, RedisRedisClientOptions, RedisRedisClassType, IORedisClientType, RedisClient, RedisMulti, RedisRedisMultiType, IORedisClientOptions, IORedisClassType, IORedisMultiType, RedisOptions, isRedisClient, isIORedisClient, } from './redis';
 export { JSONSchema, StringAnyType, StringScalarType, StringStringType, SymbolMap, SymbolMaps, SymbolRanges, Symbols, SymbolSets, } from './serializer';
 export { AggregatedData, CountByFacet, GetStatsOptions, IdsData, Measure, MeasureIds, MetricTypes, StatType, StatsType, IdsResponse, JobStats, JobStatsInput, JobStatsRange, StatsResponse, Segment, TimeSegment, } from './stats';
 export { ReclaimedMessageType, RouterConfig, StreamCode, StreamConfig, StreamData, StreamDataType, StreamError, StreamDataResponse, StreamMessage, StreamMessageMetadata, StreamProviderType, StreamRetryPolicy, StreamRole, StreamStats, StreamStatus, } from './stream';

package/build/types/index.js

@@ -1,12 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValueType = exports.trace = exports.SpanKind = exports.SpanStatusCode = exports.propagation = exports.metrics = exports.context = exports.StreamStatus = exports.StreamRole = exports.StreamDataType = exports.KeyType = exports.HookGate = exports.CollationFaultType = void 0;
+exports.ValueType = exports.trace = exports.SpanKind = exports.SpanStatusCode = exports.propagation = exports.metrics = exports.context = exports.StreamStatus = exports.StreamRole = exports.StreamDataType = exports.isIORedisClient = exports.isRedisClient = exports.KeyType = exports.HookGate = exports.CollationFaultType = void 0;
 var collator_1 = require("./collator");
 Object.defineProperty(exports, "CollationFaultType", { enumerable: true, get: function () { return collator_1.CollationFaultType; } });
 var hook_1 = require("./hook");
 Object.defineProperty(exports, "HookGate", { enumerable: true, get: function () { return hook_1.HookGate; } });
 var hotmesh_1 = require("./hotmesh");
 Object.defineProperty(exports, "KeyType", { enumerable: true, get: function () { return hotmesh_1.KeyType; } });
+var redis_1 = require("./redis");
+Object.defineProperty(exports, "isRedisClient", { enumerable: true, get: function () { return redis_1.isRedisClient; } });
+Object.defineProperty(exports, "isIORedisClient", { enumerable: true, get: function () { return redis_1.isIORedisClient; } });
 var stream_1 = require("./stream");
 Object.defineProperty(exports, "StreamDataType", { enumerable: true, get: function () { return stream_1.StreamDataType; } });
 Object.defineProperty(exports, "StreamRole", { enumerable: true, get: function () { return stream_1.StreamRole; } });

package/build/types/provider.d.ts

@@ -12,7 +12,7 @@ export interface ProviderClass {
 export interface ProviderOptions {
     [key: string]: any;
 }
-export type Providers = 'nats' | 'postgres';
+export type Providers = 'nats' | 'postgres' | 'redis' | 'ioredis';
 /**
  * A provider transaction is a set of operations that are executed
  * atomically by the provider. The transaction is created by calling

package/build/types/quorum.d.ts

@@ -94,6 +94,8 @@ export interface JobMessage extends QuorumMessageBase {
     entity?: string;
     topic: string;
     job: JobOutput;
+    /** if true, job.data is null due to payload size - subscriber should fetch via getState */
+    _ref?: boolean;
 }
 export interface ThrottleMessage extends QuorumMessageBase {
     type: 'throttle';