@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/stream/providers/nats/nats.js

@@ -0,0 +1,225 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NatsStreamService = void 0;
+const index_1 = require("../../index");
+const key_1 = require("../../../../modules/key");
+const enums_1 = require("../../../../modules/enums");
+const utils_1 = require("../../../../modules/utils");
+class NatsStreamService extends index_1.StreamService {
+    constructor(streamClient, storeClient, config = {}) {
+        super(streamClient, storeClient, config);
+        this.jetstream = streamClient.jetstream();
+    }
+    async init(namespace, appId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+        this.jsm = await this.jetstream.jetstreamManager();
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, {
+            ...params,
+            appId: this.appId,
+        });
+    }
+    transact() {
+        return {};
+    }
+    async createStream(streamName) {
+        try {
+            const config = {
+                name: streamName,
+                subjects: [`${streamName}.*`],
+                retention: 'workqueue',
+                storage: 'memory',
+                num_replicas: 1,
+            };
+            await this.jsm.streams.add(config);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`Error creating stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async deleteStream(streamName) {
+        try {
+            await this.jsm.streams.delete(streamName);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async createConsumerGroup(streamName, groupName) {
+        try {
+            const consumerConfig = {
+                durable_name: groupName,
+                deliver_group: groupName,
+                ack_policy: 'explicit',
+                ack_wait: 30 * 1000,
+                max_deliver: 10,
+            };
+            await this.jsm.consumers.add(streamName, consumerConfig);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`Error creating consumer group ${groupName} for stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async deleteConsumerGroup(streamName, groupName) {
+        try {
+            await this.jsm.consumers.delete(streamName, groupName);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting consumer group ${groupName} for stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async publishMessages(streamName, messages, options) {
+        try {
+            const publishPromises = messages.map(async (message) => {
+                const subject = `${streamName}.message`;
+                const ack = await this.jetstream.publish(subject, Buffer.from(message));
+                return ack;
+            });
+            const acks = await Promise.all(publishPromises);
+            return acks.map((ack) => ack.seq.toString());
+        }
+        catch (error) {
+            this.logger.error(`Error publishing messages to ${streamName}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async consumeMessages(streamName, groupName, consumerName, options) {
+        try {
+            const consumer = await this.jetstream.consumers.get(streamName, groupName);
+            const messages = [];
+            const fetchOptions = {
+                max_messages: options?.batchSize || 1,
+                expires: options?.blockTimeout || enums_1.HMSH_BLOCK_TIME_MS,
+            };
+            const fetchedMessages = await consumer.fetch(fetchOptions);
+            for await (const msg of fetchedMessages) {
+                messages.push({
+                    id: msg.seq.toString(),
+                    data: (0, utils_1.parseStreamMessage)(msg.string()),
+                });
+            }
+            return messages;
+        }
+        catch (error) {
+            this.logger.error(`Error consuming messages from ${streamName}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async ackAndDelete(streamName, groupName, messageIds) {
+        try {
+            await this.acknowledgeMessages(streamName, groupName, messageIds);
+            return messageIds.length;
+        }
+        catch (error) {
+            this.logger.error(`Error in ack and delete for stream ${streamName}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async acknowledgeMessages(streamName, groupName, messageIds, options) {
+        //no-op
+        return messageIds.length;
+    }
+    async deleteMessages(streamName, groupName, messageIds, options) {
+        try {
+            await Promise.all(messageIds.map((id) => this.jsm.streams.deleteMessage(streamName, parseInt(id))));
+            return messageIds.length;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting messages from ${streamName}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async retryMessages(streamName, groupName, options) {
+        return [];
+    }
+    async getStreamStats(streamName) {
+        try {
+            const info = await this.jsm.streams.info(streamName);
+            return {
+                messageCount: info.state.messages,
+            };
+        }
+        catch (error) {
+            this.logger.error(`Error getting stats for ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async getStreamDepth(streamName) {
+        try {
+            const info = await this.jsm.streams.info(streamName);
+            return info.state.messages;
+        }
+        catch (error) {
+            this.logger.error(`Error getting depth for ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async getStreamDepths(streamNames) {
+        try {
+            const results = await Promise.all(streamNames.map(async ({ stream }) => ({
+                stream,
+                depth: await this.getStreamDepth(stream),
+            })));
+            return results;
+        }
+        catch (error) {
+            this.logger.error('Error getting multiple stream depths', { error });
+            throw error;
+        }
+    }
+    async trimStream(streamName, options) {
+        try {
+            // Retrieve the current stream info
+            const streamInfo = await this.jsm.streams.info(streamName);
+            const config = { ...streamInfo.config }; // Clone to avoid direct mutation
+            // Apply new limits based on options
+            if (options.maxLen !== undefined) {
+                config.max_msgs = options.maxLen;
+            }
+            if (options.maxAge !== undefined) {
+                config.max_age = options.maxAge * 1e9; // Convert maxAge to nanoseconds
+            }
+            // Update the stream with the modified config
+            await this.jsm.streams.update(streamName, config);
+            return 0; // Trimming is applied automatically based on updated config
+        }
+        catch (error) {
+            this.logger.error(`Error trimming stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    getProviderSpecificFeatures() {
+        return {
+            supportsBatching: true,
+            supportsDeadLetterQueue: true,
+            supportsOrdering: true,
+            supportsTrimming: true,
+            supportsRetry: false,
+            supportsNotifications: false,
+            maxMessageSize: 1024 * 1024,
+            maxBatchSize: 256,
+        };
+    }
+}
+exports.NatsStreamService = NatsStreamService;

package/build/services/stream/providers/postgres/kvtables.d.ts

@@ -0,0 +1,3 @@
+import { ILogger } from '../../../logger';
+export declare function deploySchema(streamClient: any, appId: string, logger: ILogger): Promise<void>;
+export declare function getNotificationChannelName(streamName: string, groupName: string): string;

package/build/services/stream/providers/postgres/kvtables.js

@@ -0,0 +1,146 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getNotificationChannelName = exports.deploySchema = void 0;
+async function deploySchema(streamClient, appId, logger) {
+    const client = 'connect' in streamClient && 'release' in streamClient
+        ? await streamClient.connect()
+        : streamClient;
+    const releaseClient = 'release' in streamClient;
+    try {
+        const lockId = getAdvisoryLockId(appId);
+        const lockResult = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [lockId]);
+        if (lockResult.rows[0].locked) {
+            await client.query('BEGIN');
+            const schemaName = appId.replace(/[^a-zA-Z0-9_]/g, '_');
+            const tableName = `${schemaName}.streams`;
+            await createTables(client, schemaName, tableName);
+            await createNotificationTriggers(client, schemaName, tableName);
+            await client.query('COMMIT');
+            await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
+        }
+        else {
+            throw new Error('Table deployment in progress by another process.');
+        }
+    }
+    catch (error) {
+        logger.error('Error deploying schema', { error });
+        throw error;
+    }
+    finally {
+        if (releaseClient) {
+            await client.release();
+        }
+    }
+}
+exports.deploySchema = deploySchema;
+function getAdvisoryLockId(appId) {
+    return hashStringToInt(appId);
+}
+function hashStringToInt(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = (hash << 5) - hash + str.charCodeAt(i);
+        hash |= 0; // Convert to 32-bit integer
+    }
+    return Math.abs(hash);
+}
+async function createTables(client, schemaName, tableName) {
+    await client.query(`CREATE SCHEMA IF NOT EXISTS ${schemaName};`);
+    // Main table creation with partitions
+    await client.query(`
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+      id BIGSERIAL,
+      stream_name TEXT NOT NULL,
+      group_name TEXT NOT NULL DEFAULT 'ENGINE',
+      message TEXT NOT NULL,
+      created_at TIMESTAMPTZ DEFAULT NOW(),
+      reserved_at TIMESTAMPTZ,
+      reserved_by TEXT,
+      expired_at TIMESTAMPTZ,
+      PRIMARY KEY (stream_name, id)
+    ) PARTITION BY HASH (stream_name);
+  `);
+    for (let i = 0; i < 8; i++) {
+        const partitionTableName = `${schemaName}.streams_part_${i}`;
+        await client.query(`
+      CREATE TABLE IF NOT EXISTS ${partitionTableName}
+      PARTITION OF ${tableName}
+      FOR VALUES WITH (modulus 8, remainder ${i});
+    `);
+    }
+    // Index for active messages
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_streams_active_messages 
+    ON ${tableName} (group_name, stream_name, reserved_at, id)
+    WHERE reserved_at IS NULL AND expired_at IS NULL;
+  `);
+    // Optimized index for the simplified fetchMessages query
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_streams_message_fetch 
+    ON ${tableName} (stream_name, group_name, id)
+    WHERE expired_at IS NULL;
+  `);
+    // Index for expired messages
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_streams_expired_at 
+    ON ${tableName} (expired_at);
+  `);
+    // New index for stream stats optimization
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_stream_name_expired_at
+    ON ${tableName} (stream_name)
+    WHERE expired_at IS NULL;
+  `);
+    // TODO: revisit this index when solving automated cleanup
+    // Optional index for querying by creation time, if needed
+    // await client.query(`
+    //   CREATE INDEX IF NOT EXISTS idx_streams_created_at
+    //   ON ${tableName} (created_at);
+    // `);
+}
+async function createNotificationTriggers(client, schemaName, tableName) {
+    // Create the notification function
+    await client.query(`
+    CREATE OR REPLACE FUNCTION ${schemaName}.notify_new_stream_message()
+    RETURNS TRIGGER AS $$
+    DECLARE
+      channel_name TEXT;
+      payload JSON;
+    BEGIN
+      -- Create channel name: stream_{stream_name}_{group_name}
+      -- Truncate if too long (PostgreSQL channel names limited to 63 chars)
+      channel_name := 'stream_' || NEW.stream_name || '_' || NEW.group_name;
+      IF length(channel_name) > 63 THEN
+        channel_name := left(channel_name, 63);
+      END IF;
+      
+      -- Create payload with message details
+      payload := json_build_object(
+        'id', NEW.id,
+        'stream_name', NEW.stream_name,
+        'group_name', NEW.group_name,
+        'created_at', extract(epoch from NEW.created_at)
+      );
+      
+      -- Send notification
+      PERFORM pg_notify(channel_name, payload::text);
+      
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+  `);
+    // Create trigger only on the main table - it will automatically apply to all partitions
+    await client.query(`
+    DROP TRIGGER IF EXISTS notify_stream_insert ON ${tableName};
+    CREATE TRIGGER notify_stream_insert
+      AFTER INSERT ON ${tableName}
+      FOR EACH ROW
+      EXECUTE FUNCTION ${schemaName}.notify_new_stream_message();
+  `);
+}
+function getNotificationChannelName(streamName, groupName) {
+    const channelName = `stream_${streamName}_${groupName}`;
+    // PostgreSQL channel names are limited to 63 characters
+    return channelName.length > 63 ? channelName.substring(0, 63) : channelName;
+}
+exports.getNotificationChannelName = getNotificationChannelName;

package/build/services/stream/providers/postgres/postgres.d.ts

@@ -0,0 +1,107 @@
+import { ILogger } from '../../../logger';
+import { KeyType } from '../../../../modules/key';
+import { StreamService } from '../../index';
+import { KeyStoreParams, StringAnyType } from '../../../../types';
+import { PostgresClientType } from '../../../../types/postgres';
+import { PublishMessageConfig, StreamConfig, StreamMessage, StreamStats } from '../../../../types/stream';
+import { ProviderClient, ProviderTransaction } from '../../../../types/provider';
+declare class PostgresStreamService extends StreamService<PostgresClientType & ProviderClient, any> {
+    namespace: string;
+    appId: string;
+    logger: ILogger;
+    private notificationConsumers;
+    private notificationHandlerBound;
+    private fallbackIntervalId;
+    constructor(streamClient: PostgresClientType & ProviderClient, storeClient: ProviderClient, config?: StreamConfig);
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    private isNotificationsEnabled;
+    private getFallbackInterval;
+    private getNotificationTimeout;
+    private startFallbackPoller;
+    private checkForMissedMessages;
+    private handleNotification;
+    private fetchAndDeliverMessages;
+    private getConsumerKey;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    transact(): ProviderTransaction;
+    getTableName(): string;
+    safeName(appId: string): string;
+    createStream(streamName: string): Promise<boolean>;
+    deleteStream(streamName: string): Promise<boolean>;
+    createConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    deleteConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    /**
+     * `publishMessages` can be roped into a transaction by the `store`
+     * service. If so, it will add the SQL and params to the
+     * transaction. [Process Overview]: The engine keeps a reference
+     * to the `store` and `stream` providers; it asks the `store` to
+     * create a transaction and then starts adding store commands to the
+     * transaction. The engine then calls the router to publish a
+     * message using the `stream` provider (which the router keeps
+     * a reference to), and provides the transaction object.
+     * The `stream` provider then calls this method to generate
+     * the SQL and params for the transaction (but, of course, the sql
+     * is not executed until the engine calls the `exec` method on
+     * the transaction object provided by `store`).
+     *
+     * NOTE: this strategy keeps `stream` and `store` operations separate but
+     * allows calls to the stream to be roped into a single SQL transaction.
+     */
+    publishMessages(streamName: string, messages: string[], options?: PublishMessageConfig): Promise<string[] | ProviderTransaction>;
+    _publishMessages(streamName: string, messages: string[]): {
+        sql: string;
+        params: any[];
+    };
+    consumeMessages(streamName: string, groupName: string, consumerName: string, options?: {
+        batchSize?: number;
+        blockTimeout?: number;
+        autoAck?: boolean;
+        reservationTimeout?: number;
+        enableBackoff?: boolean;
+        initialBackoff?: number;
+        maxBackoff?: number;
+        maxRetries?: number;
+        enableNotifications?: boolean;
+        notificationCallback?: (messages: StreamMessage[]) => void;
+    }): Promise<StreamMessage[]>;
+    private shouldUseNotifications;
+    private setupNotificationConsumer;
+    stopNotificationConsumer(streamName: string, groupName: string): Promise<void>;
+    private fetchMessages;
+    ackAndDelete(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
+    acknowledgeMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
+    deleteMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
+    retryMessages(streamName: string, groupName: string, options?: {
+        consumerName?: string;
+        minIdleTime?: number;
+        messageIds?: string[];
+        delay?: number;
+        maxRetries?: number;
+        limit?: number;
+    }): Promise<StreamMessage[]>;
+    getStreamStats(streamName: string): Promise<StreamStats>;
+    getStreamDepth(streamName: string): Promise<number>;
+    getStreamDepths(streamNames: {
+        stream: string;
+    }[]): Promise<{
+        stream: string;
+        depth: number;
+    }[]>;
+    trimStream(streamName: string, options: {
+        maxLen?: number;
+        maxAge?: number;
+        exactLimit?: boolean;
+    }): Promise<number>;
+    getProviderSpecificFeatures(): {
+        supportsBatching: boolean;
+        supportsDeadLetterQueue: boolean;
+        supportsOrdering: boolean;
+        supportsTrimming: boolean;
+        supportsRetry: boolean;
+        supportsNotifications: boolean;
+        maxMessageSize: number;
+        maxBatchSize: number;
+    };
+    cleanup(): Promise<void>;
+}
+export { PostgresStreamService };