@hotmeshio/hotmesh 0.4.0 → 0.4.2

package/build/services/sub/index.d.ts

@@ -0,0 +1,22 @@
+import { KeyStoreParams, KeyType } from '../../modules/key';
+import { ILogger } from '../logger';
+import { SubscriptionCallback } from '../../types/quorum';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+declare abstract class SubService<ClientProvider extends ProviderClient> {
+    protected eventClient: ClientProvider;
+    protected storeClient: ProviderClient;
+    protected namespace: string;
+    protected engineId: string;
+    protected logger: ILogger;
+    protected appId: string;
+    constructor(eventClient: ClientProvider, storeClient: ProviderClient);
+    abstract init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    abstract transact(): ProviderTransaction;
+    abstract mintKey(type: KeyType, params: KeyStoreParams): string;
+    abstract subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, topic?: string): Promise<void>;
+    abstract unsubscribe(keyType: KeyType.QUORUM, appId: string, topic?: string): Promise<void>;
+    abstract psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, topic?: string): Promise<void>;
+    abstract punsubscribe(keyType: KeyType.QUORUM, appId: string, topic?: string): Promise<void>;
+    abstract publish(keyType: KeyType, message: Record<string, any>, appId: string, topic?: string): Promise<boolean>;
+}
+export { SubService };

package/build/services/sub/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SubService = void 0;
+class SubService {
+    constructor(eventClient, storeClient) {
+        this.eventClient = eventClient;
+        this.storeClient = storeClient;
+    }
+}
+exports.SubService = SubService;

package/build/services/sub/providers/nats/nats.d.ts

@@ -0,0 +1,19 @@
+import { KeyStoreParams, KeyType } from '../../../../modules/key';
+import { ILogger } from '../../../logger';
+import { SubService } from '../../index';
+import { SubscriptionCallback } from '../../../../types/quorum';
+import { NatsClientType } from '../../../../types/nats';
+import { ProviderClient, ProviderTransaction } from '../../../../types/provider';
+declare class NatsSubService extends SubService<NatsClientType & ProviderClient> {
+    private subscriptions;
+    constructor(eventClient: NatsClientType, storeClient: NatsClientType);
+    init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    transact(): ProviderTransaction;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, topic?: string): Promise<void>;
+    unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, topic?: string): Promise<void>;
+    punsubscribe(keyType: KeyType.QUORUM, appId: string, topic?: string): Promise<void>;
+    publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, topic?: string): Promise<boolean>;
+}
+export { NatsSubService };

package/build/services/sub/providers/nats/nats.js

@@ -0,0 +1,105 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NatsSubService = void 0;
+const key_1 = require("../../../../modules/key");
+const index_1 = require("../../index");
+class NatsSubService extends index_1.SubService {
+    constructor(eventClient, storeClient) {
+        super(eventClient, storeClient);
+        this.subscriptions = new Map();
+    }
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.engineId = engineId;
+    }
+    transact() {
+        // NATS does not support transactions like Redis.
+        // Return an empty object or throw an error if not supported.
+        return {};
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, params)
+            .replace(/:/g, '.')
+            .replace(/\.$/g, '.X');
+    }
+    async subscribe(keyType, callback, appId, topic) {
+        const subject = this.mintKey(keyType, { appId, engineId: topic });
+        const subscription = this.eventClient.subscribe(subject);
+        this.subscriptions.set(subject, subscription);
+        this.logger.debug(`nats-subscribe ${subject}`);
+        (async () => {
+            for await (const msg of subscription) {
+                try {
+                    const payload = JSON.parse(msg.data.toString());
+                    callback(subject, payload);
+                }
+                catch (e) {
+                    this.logger.error(`nats-subscribe-message-parse-error: ${msg.data.toString()}`, e);
+                }
+            }
+        })().catch((err) => {
+            this.logger.error(`nats-subscribe-error ${subject}`, err);
+        });
+    }
+    async unsubscribe(keyType, appId, engineId) {
+        const subject = this.mintKey(keyType, { appId, engineId });
+        const subscription = this.subscriptions.get(subject);
+        if (subscription) {
+            subscription.unsubscribe();
+            this.subscriptions.delete(subject);
+            this.logger.debug(`nats-unsubscribe ${subject}`);
+        }
+        else {
+            this.logger.warn(`nats-unsubscribe-error ${subject}`);
+        }
+    }
+    async psubscribe(keyType, callback, appId, topic) {
+        const subject = this.mintKey(keyType, { appId, engineId: topic });
+        const subscription = this.eventClient.subscribe(subject);
+        this.subscriptions.set(subject, subscription);
+        this.logger.debug(`nats-psubscribe ${subject}`);
+        (async () => {
+            for await (const msg of subscription) {
+                try {
+                    const payload = JSON.parse(msg.data.toString());
+                    callback(msg.subject, payload);
+                }
+                catch (e) {
+                    this.logger.error(`nats-parse-psubscription-message-error ${msg.data.toString()}`, e);
+                }
+            }
+        })().catch((err) => {
+            this.logger.error(`nats-pattern-psubscription-error ${subject}`, err);
+        });
+    }
+    async punsubscribe(keyType, appId, topic) {
+        const subject = this.mintKey(keyType, { appId, engineId: topic });
+        const subscription = this.subscriptions.get(subject);
+        if (subscription) {
+            subscription.unsubscribe();
+            this.subscriptions.delete(subject);
+            this.logger.debug(`nats-punsubscribe ${subject}`);
+        }
+        else {
+            this.logger.warn(`nats-punsubscribe-error ${subject}`);
+        }
+    }
+    async publish(keyType, message, appId, topic) {
+        const subject = this.mintKey(keyType, { appId, engineId: topic });
+        try {
+            // Use the storeClient for publishing if necessary
+            this.storeClient.publish(subject, JSON.stringify(message));
+            this.logger.debug(`nats-publish ${subject}`);
+            return true;
+        }
+        catch (err) {
+            this.logger.error(`nats-publish-error ${subject}`, err);
+            return false;
+        }
+    }
+}
+exports.NatsSubService = NatsSubService;

package/build/services/sub/providers/postgres/postgres.d.ts

@@ -0,0 +1,19 @@
+import { KeyStoreParams, KeyType } from '../../../../modules/key';
+import { ILogger } from '../../../logger';
+import { SubService } from '../../index';
+import { SubscriptionCallback } from '../../../../types/quorum';
+import { ProviderClient, ProviderTransaction } from '../../../../types/provider';
+import { PostgresClientType } from '../../../../types/postgres';
+declare class PostgresSubService extends SubService<PostgresClientType & ProviderClient> {
+    constructor(eventClient: PostgresClientType & ProviderClient, storeClient?: PostgresClientType & ProviderClient);
+    init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    transact(): ProviderTransaction;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    mintSafeKey(type: KeyType, params: KeyStoreParams): [string, string];
+    subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, topic?: string): Promise<void>;
+    unsubscribe(keyType: KeyType.QUORUM, appId: string, topic?: string): Promise<void>;
+    publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, topic?: string): Promise<boolean>;
+    psubscribe(): Promise<void>;
+    punsubscribe(): Promise<void>;
+}
+export { PostgresSubService };

package/build/services/sub/providers/postgres/postgres.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresSubService = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const key_1 = require("../../../../modules/key");
+const index_1 = require("../../index");
+class PostgresSubService extends index_1.SubService {
+    constructor(eventClient, storeClient) {
+        super(eventClient, storeClient);
+    }
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.engineId = engineId;
+    }
+    transact() {
+        throw new Error('Transactions are not supported in lightweight pub/sub');
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('Namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, params);
+    }
+    mintSafeKey(type, params) {
+        const originalKey = this.mintKey(type, params);
+        if (originalKey.length <= 63) {
+            return [originalKey, originalKey];
+        }
+        const { appId = '', engineId = '' } = params;
+        const baseKey = `${this.namespace}:${appId}:${type}`;
+        const maxHashLength = 63 - baseKey.length - 1; // Reserve space for `:` delimiter
+        const engineIdHash = crypto_1.default
+            .createHash('sha256')
+            .update(engineId)
+            .digest('hex')
+            .substring(0, maxHashLength);
+        const safeKey = `${baseKey}:${engineIdHash}`;
+        return [originalKey, safeKey];
+    }
+    async subscribe(keyType, callback, appId, topic) {
+        const [originalKey, safeKey] = this.mintSafeKey(keyType, {
+            appId,
+            engineId: topic,
+        });
+        // Start listening to the safe topic
+        await this.eventClient.query(`LISTEN "${safeKey}"`);
+        this.logger.debug(`postgres-subscribe`, { originalKey, safeKey });
+        // Set up the notification handler
+        this.eventClient.on('notification', (msg) => {
+            if (msg.channel === safeKey) {
+                try {
+                    const payload = JSON.parse(msg.payload || '{}');
+                    callback(safeKey, payload);
+                }
+                catch (err) {
+                    this.logger.error(`Error parsing message for topic ${safeKey}:`, err);
+                }
+            }
+        });
+    }
+    async unsubscribe(keyType, appId, topic) {
+        const [originalKey, safeKey] = this.mintSafeKey(keyType, {
+            appId,
+            engineId: topic,
+        });
+        // Stop listening to the safe topic
+        await this.eventClient.query(`UNLISTEN "${safeKey}"`);
+        this.logger.debug(`postgres-subscribe`, { originalKey, safeKey });
+    }
+    async publish(keyType, message, appId, topic) {
+        const [originalKey, safeKey] = this.mintSafeKey(keyType, {
+            appId,
+            engineId: topic,
+        });
+        // Publish the message using the safe topic
+        const payload = JSON.stringify(message).replace(/'/g, "''");
+        await this.storeClient.query(`NOTIFY "${safeKey}", '${payload}'`);
+        this.logger.debug(`postgres-publish`, { originalKey, safeKey });
+        return true;
+    }
+    async psubscribe() {
+        throw new Error('Pattern subscriptions are not supported in PostgreSQL');
+    }
+    async punsubscribe() {
+        throw new Error('Pattern subscriptions are not supported in PostgreSQL');
+    }
+}
+exports.PostgresSubService = PostgresSubService;

package/build/services/sub/providers/redis/ioredis.d.ts

@@ -0,0 +1,17 @@
+import { KeyStoreParams, KeyType } from '../../../../modules/key';
+import { ILogger } from '../../../logger';
+import { SubService } from '../../index';
+import { IORedisClientType as RedisClientType, IORedisMultiType as RedisMultiType } from '../../../../types/redis';
+import { SubscriptionCallback } from '../../../../types/quorum';
+declare class IORedisSubService extends SubService<RedisClientType> {
+    constructor(eventClient: RedisClientType, storeClient: RedisClientType);
+    init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    transact(): RedisMultiType;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean>;
+}
+export { IORedisSubService };

package/build/services/sub/providers/redis/ioredis.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisSubService = void 0;
+const key_1 = require("../../../../modules/key");
+const index_1 = require("../../index");
+class IORedisSubService extends index_1.SubService {
+    constructor(eventClient, storeClient) {
+        super(eventClient, storeClient);
+    }
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.engineId = engineId;
+    }
+    transact() {
+        return this.eventClient.multi();
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, params);
+    }
+    async subscribe(keyType, callback, appId, engineId) {
+        const self = this;
+        const topic = this.mintKey(keyType, { appId, engineId });
+        await this.eventClient.subscribe(topic, (err) => {
+            if (err) {
+                self.logger.error(`Error subscribing to: ${topic}`, err);
+            }
+        });
+        this.eventClient.on('message', (channel, message) => {
+            if (channel === topic) {
+                try {
+                    const payload = JSON.parse(message);
+                    callback(topic, payload);
+                }
+                catch (e) {
+                    self.logger.error(`Error parsing message: ${message}`, e);
+                }
+            }
+        });
+    }
+    async unsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        await this.eventClient.unsubscribe(topic);
+    }
+    async psubscribe(keyType, callback, appId, engineId) {
+        const self = this;
+        const topic = this.mintKey(keyType, { appId, engineId });
+        await this.eventClient.psubscribe(topic, (err) => {
+            if (err) {
+                self.logger.error(`Error subscribing to: ${topic}`, err);
+            }
+        });
+        this.eventClient.on('pmessage', (pattern, channel, message) => {
+            if (pattern === topic) {
+                try {
+                    const payload = JSON.parse(message);
+                    callback(channel, payload);
+                }
+                catch (e) {
+                    self.logger.error(`Error parsing message: ${message}`, e);
+                }
+            }
+        });
+    }
+    async punsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        await this.eventClient.punsubscribe(topic);
+    }
+    async publish(keyType, message, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        //NOTE: `storeClient.publish` is used,
+        //      because a Redis connection with subscriptions
+        //      may not publish (is read only).
+        const status = await this.storeClient.publish(topic, JSON.stringify(message));
+        return status === 1;
+    }
+}
+exports.IORedisSubService = IORedisSubService;

package/build/services/sub/providers/redis/redis.d.ts

@@ -0,0 +1,17 @@
+import { KeyStoreParams, KeyType } from '../../../../modules/key';
+import { ILogger } from '../../../logger';
+import { SubService } from '../../index';
+import { RedisRedisClientType as ClientProvider, RedisRedisMultiType as TransactionProvider } from '../../../../types/redis';
+import { SubscriptionCallback } from '../../../../types/quorum';
+declare class RedisSubService extends SubService<ClientProvider> {
+    constructor(eventClient: ClientProvider, storeClient: ClientProvider);
+    init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+    transact(): TransactionProvider;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+    punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+    publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean>;
+}
+export { RedisSubService };

package/build/services/sub/providers/redis/redis.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisSubService = void 0;
+const key_1 = require("../../../../modules/key");
+const index_1 = require("../../index");
+class RedisSubService extends index_1.SubService {
+    constructor(eventClient, storeClient) {
+        super(eventClient, storeClient);
+    }
+    async init(namespace = key_1.HMNS, appId, engineId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.engineId = engineId;
+    }
+    transact() {
+        const multi = this.eventClient.multi();
+        return multi;
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, params);
+    }
+    async subscribe(keyType, callback, appId, engineId) {
+        if (this.eventClient) {
+            const self = this;
+            const topic = this.mintKey(keyType, { appId, engineId });
+            await this.eventClient.subscribe(topic, (message) => {
+                try {
+                    const payload = JSON.parse(message);
+                    callback(topic, payload);
+                }
+                catch (e) {
+                    self.logger.error(`Error parsing message: ${message}`, e);
+                }
+            });
+        }
+    }
+    async unsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        await this.eventClient.unsubscribe(topic);
+    }
+    async psubscribe(keyType, callback, appId, engineId) {
+        if (this.eventClient) {
+            const self = this;
+            const topic = this.mintKey(keyType, { appId, engineId });
+            await this.eventClient.pSubscribe(topic, (message, channel) => {
+                try {
+                    const payload = JSON.parse(message);
+                    callback(channel, payload);
+                }
+                catch (e) {
+                    self.logger.error(`Error parsing message: ${message}`, e);
+                }
+            });
+        }
+    }
+    async punsubscribe(keyType, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        await this.eventClient.pUnsubscribe(topic);
+    }
+    async publish(keyType, message, appId, engineId) {
+        const topic = this.mintKey(keyType, { appId, engineId });
+        //NOTE: `storeClient.publish` is used,
+        //      because a Redis connection with subscriptions
+        //      may not publish (is read only).
+        const status = await this.storeClient.publish(topic, JSON.stringify(message));
+        return status > 0;
+    }
+}
+exports.RedisSubService = RedisSubService;

package/build/services/task/index.d.ts

@@ -0,0 +1,48 @@
+/// <reference types="node" />
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { HookInterface, HookRule } from '../../types/hook';
+import { JobCompletionOptions, JobState } from '../../types/job';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { WorkListTaskType } from '../../types/task';
+declare class TaskService {
+    store: StoreService<ProviderClient, ProviderTransaction>;
+    logger: ILogger;
+    cleanupTimeout: NodeJS.Timeout | null;
+    isScout: boolean;
+    errorCount: number;
+    constructor(store: StoreService<ProviderClient, ProviderTransaction>, logger: ILogger);
+    processWebHooks(hookEventCallback: HookInterface): Promise<void>;
+    enqueueWorkItems(keys: string[]): Promise<void>;
+    registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
+    registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, inSeconds: number, dad: string, transaction?: ProviderTransaction): Promise<void>;
+    /**
+     * Should this engine instance play the role of 'scout' on behalf
+     * of the entire quorum? The scout role is responsible for processing
+     * task lists on behalf of the collective.
+     */
+    shouldScout(): Promise<boolean>;
+    /**
+     * Callback handler that takes an item from a work list and
+     * processes according to its type
+     */
+    processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
+    cancelCleanup(): void;
+    getHookRule(topic: string): Promise<HookRule | undefined>;
+    registerWebHook(topic: string, context: JobState, dad: string, expire: number, transaction?: ProviderTransaction): Promise<string>;
+    processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<[string, string, string, string] | undefined>;
+    deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
+    /**
+     * Enhanced processTimeHooks that uses notifications for PostgreSQL stores
+     */
+    processTimeHooksWithNotifications(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>): Promise<void>;
+    /**
+     * Check if this is a PostgreSQL store
+     */
+    private isPostgresStore;
+    /**
+     * Check if the store supports notifications
+     */
+    private supportsNotifications;
+}
+export { TaskService };