@hotmeshio/hotmesh 0.3.32 → 0.4.1

package/build/services/stream/providers/postgres/kvtables.d.ts

@@ -1,2 +1,3 @@
 import { ILogger } from '../../../logger';
 export declare function deploySchema(streamClient: any, appId: string, logger: ILogger): Promise<void>;
+export declare function getNotificationChannelName(streamName: string, groupName: string): string;

package/build/services/stream/providers/postgres/kvtables.js

@@ -1 +1,146 @@
-'use strict';const q=b;(function(c,d){const m=b,e=c();while(!![]){try{const f=-parseInt(m(0x188))/0x1+parseInt(m(0x194))/0x2+parseInt(m(0x191))/0x3*(parseInt(m(0x183))/0x4)+parseInt(m(0x17c))/0x5*(-parseInt(m(0x18f))/0x6)+parseInt(m(0x18e))/0x7+-parseInt(m(0x18d))/0x8+-parseInt(m(0x195))/0x9*(-parseInt(m(0x17d))/0xa);if(f===d)break;else e['push'](e['shift']());}catch(g){e['push'](e['shift']());}}}(a,0xeddea));function b(c,d){const e=a();return b=function(f,g){f=f-0x17c;let h=e[f];return h;},b(c,d);}async function deploySchema(c,d,f){const n=b,g=n(0x181)in c&&n(0x17e)in c?await c[n(0x181)]():c,h=n(0x17e)in c;try{const i=getAdvisoryLockId(d);if(!(await g[n(0x192)](n(0x19d),[i]))[n(0x198)][0x0][n(0x189)])throw new Error('Table\x20deployment\x20in\x20progress\x20by\x20another\x20process.');{await g[n(0x192)](n(0x190));const j=d[n(0x17f)](/[^a-zA-Z0-9_]/g,'_'),k=j+'.streams';await createTables(g,j,k),await g[n(0x192)]('COMMIT'),await g['query'](n(0x180),[i]);}}catch(l){throw f[n(0x199)](n(0x186),{'error':l}),l;}finally{h&&await g[n(0x17e)]();}}function a(){const r=['rows','error','\x0a\x20\x20\x20\x20\x20\x20FOR\x20VALUES\x20WITH\x20(modulus\x208,\x20remainder\x20','__esModule','CREATE\x20SCHEMA\x20IF\x20NOT\x20EXISTS\x20','SELECT\x20pg_try_advisory_lock($1)\x20AS\x20locked','abs','\x20(\x0a\x20\x20\x20\x20\x20\x20id\x20BIGSERIAL,\x0a\x20\x20\x20\x20\x20\x20stream_name\x20TEXT\x20NOT\x20NULL,\x0a\x20\x20\x20\x20\x20\x20group_name\x20TEXT\x20NOT\x20NULL\x20DEFAULT\x20\x27ENGINE\x27,\x0a\x20\x20\x20\x20\x20\x20message\x20TEXT\x20NOT\x20NULL,\x0a\x20\x20\x20\x20\x20\x20created_at\x20TIMESTAMPTZ\x20DEFAULT\x20NOW(),\x0a\x20\x20\x20\x20\x20\x20reserved_at\x20TIMESTAMPTZ,\x0a\x20\x20\x20\x20\x20\x20reserved_by\x20TEXT,\x0a\x20\x20\x20\x20\x20\x20expired_at\x20TIMESTAMPTZ,\x0a\x20\x20\x20\x20\x20\x20PRIMARY\x20KEY\x20(stream_name,\x20id)\x0a\x20\x20\x20\x20)\x20PARTITION\x20BY\x20HASH\x20(stream_name);\x0a\x20\x20','445fJhQsm','10FGSthe','release','replace','SELECT\x20pg_advisory_unlock($1)','connect','\x0a\x20\x20\x20\x20CREATE\x20INDEX\x20IF\x20NOT\x20EXISTS\x20idx_streams_expired_at\x20\x0a\x20\x20\x20\x20ON\x20','22720zAeqOr','deploySchema','\x0a\x20\x20\x20\x20CREATE\x20TABLE\x20IF\x20NOT\x20EXISTS\x20','Error\x20deploying\x20schema','length','615963umCfKL','locked','\x20(group_name,\x20stream_name,\x20reserved_at,\x20id)\x0a\x20\x20\x20\x20WHERE\x20reserved_at\x20IS\x20NULL\x20AND\x20expired_at\x20IS\x20NULL;\x0a\x20\x20','charCodeAt',');\x0a\x20\x20\x20\x20','5679472Hhquqq','3812613sejpXO','90084YBtADi','BEGIN','582JXhHOY','query','\x0a\x20\x20\x20\x20\x20\x20CREATE\x20TABLE\x20IF\x20NOT\x20EXISTS\x20','2143808RWLfnk','8261766NVUIKN','defineProperty','.streams_part_'];a=function(){return r;};return a();}function getAdvisoryLockId(c){return hashStringToInt(c);}function hashStringToInt(c){const o=b;let d=0x0;for(let f=0x0;f<c[o(0x187)];f++)d=(d<<0x5)-d+c[o(0x18b)](f),d|=0x0;return Math[o(0x19e)](d);}async function createTables(c,d,f){const p=b;await c[p(0x192)](p(0x19c)+d+';'),await c[p(0x192)](p(0x185)+f+p(0x19f));for(let g=0x0;g<0x8;g++){const h=d+p(0x197)+g;await c['query'](p(0x193)+h+'\x0a\x20\x20\x20\x20\x20\x20PARTITION\x20OF\x20'+f+p(0x19a)+g+p(0x18c));}await c['query']('\x0a\x20\x20\x20\x20CREATE\x20INDEX\x20IF\x20NOT\x20EXISTS\x20idx_streams_active_messages\x20\x0a\x20\x20\x20\x20ON\x20'+f+p(0x18a)),await c['query'](p(0x182)+f+'\x20(expired_at);\x0a\x20\x20'),await c[p(0x192)]('\x0a\x20\x20\x20\x20CREATE\x20INDEX\x20IF\x20NOT\x20EXISTS\x20idx_stream_name_expired_at\x0a\x20\x20\x20\x20ON\x20'+f+'\x20(stream_name)\x0a\x20\x20\x20\x20WHERE\x20expired_at\x20IS\x20NULL;\x0a\x20\x20');}Object[q(0x196)](exports,q(0x19b),{'value':!0x0}),exports['deploySchema']=void 0x0,exports[q(0x184)]=deploySchema;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getNotificationChannelName = exports.deploySchema = void 0;
+async function deploySchema(streamClient, appId, logger) {
+    const client = 'connect' in streamClient && 'release' in streamClient
+        ? await streamClient.connect()
+        : streamClient;
+    const releaseClient = 'release' in streamClient;
+    try {
+        const lockId = getAdvisoryLockId(appId);
+        const lockResult = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [lockId]);
+        if (lockResult.rows[0].locked) {
+            await client.query('BEGIN');
+            const schemaName = appId.replace(/[^a-zA-Z0-9_]/g, '_');
+            const tableName = `${schemaName}.streams`;
+            await createTables(client, schemaName, tableName);
+            await createNotificationTriggers(client, schemaName, tableName);
+            await client.query('COMMIT');
+            await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
+        }
+        else {
+            throw new Error('Table deployment in progress by another process.');
+        }
+    }
+    catch (error) {
+        logger.error('Error deploying schema', { error });
+        throw error;
+    }
+    finally {
+        if (releaseClient) {
+            await client.release();
+        }
+    }
+}
+exports.deploySchema = deploySchema;
+function getAdvisoryLockId(appId) {
+    return hashStringToInt(appId);
+}
+function hashStringToInt(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = (hash << 5) - hash + str.charCodeAt(i);
+        hash |= 0; // Convert to 32-bit integer
+    }
+    return Math.abs(hash);
+}
+async function createTables(client, schemaName, tableName) {
+    await client.query(`CREATE SCHEMA IF NOT EXISTS ${schemaName};`);
+    // Main table creation with partitions
+    await client.query(`
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+      id BIGSERIAL,
+      stream_name TEXT NOT NULL,
+      group_name TEXT NOT NULL DEFAULT 'ENGINE',
+      message TEXT NOT NULL,
+      created_at TIMESTAMPTZ DEFAULT NOW(),
+      reserved_at TIMESTAMPTZ,
+      reserved_by TEXT,
+      expired_at TIMESTAMPTZ,
+      PRIMARY KEY (stream_name, id)
+    ) PARTITION BY HASH (stream_name);
+  `);
+    for (let i = 0; i < 8; i++) {
+        const partitionTableName = `${schemaName}.streams_part_${i}`;
+        await client.query(`
+      CREATE TABLE IF NOT EXISTS ${partitionTableName}
+      PARTITION OF ${tableName}
+      FOR VALUES WITH (modulus 8, remainder ${i});
+    `);
+    }
+    // Index for active messages
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_streams_active_messages 
+    ON ${tableName} (group_name, stream_name, reserved_at, id)
+    WHERE reserved_at IS NULL AND expired_at IS NULL;
+  `);
+    // Optimized index for the simplified fetchMessages query
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_streams_message_fetch 
+    ON ${tableName} (stream_name, group_name, id)
+    WHERE expired_at IS NULL;
+  `);
+    // Index for expired messages
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_streams_expired_at 
+    ON ${tableName} (expired_at);
+  `);
+    // New index for stream stats optimization
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS idx_stream_name_expired_at
+    ON ${tableName} (stream_name)
+    WHERE expired_at IS NULL;
+  `);
+    // TODO: revisit this index when solving automated cleanup
+    // Optional index for querying by creation time, if needed
+    // await client.query(`
+    //   CREATE INDEX IF NOT EXISTS idx_streams_created_at
+    //   ON ${tableName} (created_at);
+    // `);
+}
+async function createNotificationTriggers(client, schemaName, tableName) {
+    // Create the notification function
+    await client.query(`
+    CREATE OR REPLACE FUNCTION ${schemaName}.notify_new_stream_message()
+    RETURNS TRIGGER AS $$
+    DECLARE
+      channel_name TEXT;
+      payload JSON;
+    BEGIN
+      -- Create channel name: stream_{stream_name}_{group_name}
+      -- Truncate if too long (PostgreSQL channel names limited to 63 chars)
+      channel_name := 'stream_' || NEW.stream_name || '_' || NEW.group_name;
+      IF length(channel_name) > 63 THEN
+        channel_name := left(channel_name, 63);
+      END IF;
+      
+      -- Create payload with message details
+      payload := json_build_object(
+        'id', NEW.id,
+        'stream_name', NEW.stream_name,
+        'group_name', NEW.group_name,
+        'created_at', extract(epoch from NEW.created_at)
+      );
+      
+      -- Send notification
+      PERFORM pg_notify(channel_name, payload::text);
+      
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+  `);
+    // Create trigger only on the main table - it will automatically apply to all partitions
+    await client.query(`
+    DROP TRIGGER IF EXISTS notify_stream_insert ON ${tableName};
+    CREATE TRIGGER notify_stream_insert
+      AFTER INSERT ON ${tableName}
+      FOR EACH ROW
+      EXECUTE FUNCTION ${schemaName}.notify_new_stream_message();
+  `);
+}
+function getNotificationChannelName(streamName, groupName) {
+    const channelName = `stream_${streamName}_${groupName}`;
+    // PostgreSQL channel names are limited to 63 characters
+    return channelName.length > 63 ? channelName.substring(0, 63) : channelName;
+}
+exports.getNotificationChannelName = getNotificationChannelName;

package/build/services/stream/providers/postgres/postgres.d.ts

@@ -9,8 +9,19 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
     namespace: string;
     appId: string;
     logger: ILogger;
+    private notificationConsumers;
+    private notificationHandlerBound;
+    private fallbackIntervalId;
     constructor(streamClient: PostgresClientType & ProviderClient, storeClient: ProviderClient, config?: StreamConfig);
     init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    private isNotificationsEnabled;
+    private getFallbackInterval;
+    private getNotificationTimeout;
+    private startFallbackPoller;
+    private checkForMissedMessages;
+    private handleNotification;
+    private fetchAndDeliverMessages;
+    private getConsumerKey;
     mintKey(type: KeyType, params: KeyStoreParams): string;
     transact(): ProviderTransaction;
     getTableName(): string;
@@ -50,7 +61,13 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
         initialBackoff?: number;
         maxBackoff?: number;
         maxRetries?: number;
+        enableNotifications?: boolean;
+        notificationCallback?: (messages: StreamMessage[]) => void;
     }): Promise<StreamMessage[]>;
+    private shouldUseNotifications;
+    private setupNotificationConsumer;
+    stopNotificationConsumer(streamName: string, groupName: string): Promise<void>;
+    private fetchMessages;
     ackAndDelete(streamName: string, groupName: string, messageIds: string[]): Promise<number>;
     acknowledgeMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
     deleteMessages(streamName: string, groupName: string, messageIds: string[], options?: StringAnyType): Promise<number>;
@@ -81,8 +98,10 @@ declare class PostgresStreamService extends StreamService<PostgresClientType & P
         supportsOrdering: boolean;
         supportsTrimming: boolean;
         supportsRetry: boolean;
+        supportsNotifications: boolean;
         maxMessageSize: number;
         maxBatchSize: number;
     };
+    cleanup(): Promise<void>;
 }
 export { PostgresStreamService };