@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/store/providers/postgres/kvtables.js

@@ -0,0 +1,441 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KVTables = void 0;
+const enums_1 = require("../../../../modules/enums");
+const utils_1 = require("../../../../modules/utils");
+const KVTables = (context) => ({
+    /**
+     * Deploys the necessary tables with the specified naming strategy.
+     * @param appName - The name of the application.
+     */
+    async deploy(appName) {
+        const transactionClient = context.pgClient;
+        let client;
+        let releaseClient = false;
+        if (transactionClient?.totalCount !== undefined &&
+            transactionClient?.idleCount !== undefined) {
+            // It's a Pool, need to acquire a client
+            client = await transactionClient.connect();
+            releaseClient = true;
+        }
+        else {
+            // Assume it's a connected Client
+            client = transactionClient;
+        }
+        try {
+            // Acquire advisory lock
+            const lockId = this.getAdvisoryLockId(appName);
+            const lockResult = await client.query('SELECT pg_try_advisory_lock($1) AS locked', [lockId]);
+            if (lockResult.rows[0].locked) {
+                // Begin transaction
+                await client.query('BEGIN');
+                // Check and create tables
+                const tablesExist = await this.checkIfTablesExist(client, appName);
+                if (!tablesExist) {
+                    await this.createTables(client, appName);
+                }
+                // Commit transaction
+                await client.query('COMMIT');
+                // Release the lock
+                await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
+            }
+            else {
+                // Release the client before waiting
+                if (releaseClient && client.release) {
+                    await client.release();
+                    releaseClient = false;
+                }
+                // Wait for the deploy process to complete
+                await this.waitForTablesCreation(lockId, appName);
+            }
+        }
+        catch (error) {
+            console.error(error);
+            context.logger.error('Error deploying tables', { error });
+            throw error;
+        }
+        finally {
+            if (releaseClient && client.release) {
+                await client.release();
+            }
+        }
+    },
+    getAdvisoryLockId(appName) {
+        return this.hashStringToInt(appName);
+    },
+    hashStringToInt(str) {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            hash = (hash << 5) - hash + str.charCodeAt(i);
+            hash |= 0; // Convert to 32-bit integer
+        }
+        return Math.abs(hash);
+    },
+    async waitForTablesCreation(lockId, appName) {
+        let retries = 0;
+        const maxRetries = Math.round(enums_1.HMSH_DEPLOYMENT_DELAY / enums_1.HMSH_DEPLOYMENT_PAUSE);
+        while (retries < maxRetries) {
+            await (0, utils_1.sleepFor)(enums_1.HMSH_DEPLOYMENT_PAUSE);
+            let client;
+            let releaseClient = false;
+            const transactionClient = context.pgClient;
+            if (transactionClient?.totalCount !== undefined &&
+                transactionClient?.idleCount !== undefined) {
+                // It's a Pool, need to acquire a client
+                client = await transactionClient.connect();
+                releaseClient = true;
+            }
+            else {
+                // Assume it's a connected Client
+                client = transactionClient;
+            }
+            try {
+                const lockCheck = await client.query("SELECT NOT EXISTS (SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND objid = $1::bigint) AS unlocked", [lockId]);
+                if (lockCheck.rows[0].unlocked) {
+                    // Lock has been released, tables should exist now
+                    const tablesExist = await this.checkIfTablesExist(client, appName);
+                    if (tablesExist) {
+                        return;
+                    }
+                }
+            }
+            finally {
+                if (releaseClient && client.release) {
+                    await client.release();
+                }
+            }
+            retries++;
+        }
+        console.error('table-create-timeout', { appName });
+        throw new Error('Timeout waiting for table creation');
+    },
+    async checkIfTablesExist(client, appName) {
+        const tableNames = this.getTableNames(appName);
+        const checkTablePromises = tableNames.map((tableName) => client.query(`SELECT to_regclass('${tableName}') AS table`));
+        const results = await Promise.all(checkTablePromises);
+        return results.every((res) => res.rows[0].table !== null);
+    },
+    async createTables(client, appName) {
+        try {
+            await client.query('BEGIN');
+            const schemaName = context.storeClient.safeName(appName);
+            await client.query(`CREATE SCHEMA IF NOT EXISTS ${schemaName};`);
+            const tableDefinitions = this.getTableDefinitions(appName);
+            for (const tableDef of tableDefinitions) {
+                const fullTableName = `${tableDef.schema}.${tableDef.name}`;
+                switch (tableDef.type) {
+                    case 'string':
+                        await client.query(`
+              CREATE TABLE IF NOT EXISTS ${fullTableName} (
+                key TEXT PRIMARY KEY,
+                value TEXT,
+                expiry TIMESTAMP WITH TIME ZONE
+              );
+            `);
+                        break;
+                    case 'hash':
+                        await client.query(`
+              CREATE TABLE IF NOT EXISTS ${fullTableName} (
+                key TEXT NOT NULL,
+                field TEXT NOT NULL,
+                value TEXT,
+                expiry TIMESTAMP WITH TIME ZONE,
+                PRIMARY KEY (key, field)
+              );
+            `);
+                        break;
+                    case 'jobhash':
+                        // Create the enum type in the schema
+                        await client.query(`
+              DO $$
+              BEGIN
+                IF NOT EXISTS (
+                  SELECT 1 FROM pg_type t
+                  JOIN pg_namespace n ON n.oid = t.typnamespace
+                  WHERE t.typname = 'type_enum' AND n.nspname = '${schemaName}'
+                ) THEN
+                  CREATE TYPE ${schemaName}.type_enum AS ENUM ('jmark', 'hmark', 'status', 'jdata', 'adata', 'udata', 'other');
+                END IF;
+              END$$;
+            `);
+                        // Create the main jobs table with partitioning on id
+                        await client.query(`
+              CREATE TABLE IF NOT EXISTS ${fullTableName} (
+                id UUID DEFAULT gen_random_uuid(),
+                key TEXT NOT NULL,
+                entity TEXT,
+                status INTEGER NOT NULL,
+                context JSONB DEFAULT '{}',
+                created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+                updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+                expired_at TIMESTAMP WITH TIME ZONE,
+                is_live BOOLEAN DEFAULT TRUE,
+                PRIMARY KEY (id)
+              ) PARTITION BY HASH (id);
+            `);
+                        // Create GIN index for full JSONB search
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_context_gin 
+              ON ${fullTableName} USING GIN (context);
+            `);
+                        // Create partitions using a DO block
+                        await client.query(`
+              DO $$
+              BEGIN
+                FOR i IN 0..7 LOOP
+                  EXECUTE format(
+                    'CREATE TABLE IF NOT EXISTS ${fullTableName}_part_%s PARTITION OF ${fullTableName} 
+                    FOR VALUES WITH (modulus 8, remainder %s)',
+                    i, i
+                  );
+                END LOOP;
+              END$$;
+            `);
+                        // Create optimized indexes
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_expired_at 
+              ON ${fullTableName} (key, expired_at) INCLUDE (is_live);
+            `);
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_entity_status 
+              ON ${fullTableName} (entity, status);
+            `);
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_expired_at 
+              ON ${fullTableName} (expired_at);
+            `);
+                        // Create function to update is_live flag in the schema
+                        await client.query(`
+              CREATE OR REPLACE FUNCTION ${schemaName}.update_is_live()
+              RETURNS TRIGGER AS $$
+              BEGIN
+                NEW.is_live := NEW.expired_at IS NULL OR NEW.expired_at > NOW();
+                RETURN NEW;
+              END;
+              $$ LANGUAGE plpgsql;
+            `);
+                        // Create trigger for is_live updates
+                        await client.query(`
+              CREATE TRIGGER trg_update_is_live
+              BEFORE INSERT OR UPDATE ON ${fullTableName}
+              FOR EACH ROW EXECUTE PROCEDURE ${schemaName}.update_is_live();
+            `);
+                        // Create function to enforce uniqueness of live jobs
+                        await client.query(`
+              CREATE OR REPLACE FUNCTION ${schemaName}.enforce_live_job_uniqueness()
+              RETURNS TRIGGER AS $$
+              BEGIN
+                IF (NEW.expired_at IS NULL OR NEW.expired_at > NOW()) THEN
+                  PERFORM pg_advisory_xact_lock(hashtextextended(NEW.key, 0));
+                  IF EXISTS (
+                    SELECT 1 FROM ${fullTableName}
+                    WHERE key = NEW.key 
+                    AND (expired_at IS NULL OR expired_at > NOW())
+                    AND id <> NEW.id
+                  ) THEN
+                    RAISE EXCEPTION 'A live job with key % already exists.', NEW.key;
+                  END IF;
+                END IF;
+                RETURN NEW;
+              END;
+              $$ LANGUAGE plpgsql;
+            `);
+                        // Create trigger for uniqueness enforcement
+                        await client.query(`
+              CREATE TRIGGER trg_enforce_live_job_uniqueness
+              BEFORE INSERT OR UPDATE ON ${fullTableName}
+              FOR EACH ROW EXECUTE PROCEDURE ${schemaName}.enforce_live_job_uniqueness();
+            `);
+                        // Create the attributes table with partitioning
+                        const attributesTableName = `${fullTableName}_attributes`;
+                        await client.query(`
+              CREATE TABLE IF NOT EXISTS ${attributesTableName} (
+                job_id UUID NOT NULL,
+                field TEXT NOT NULL,
+                value TEXT,
+                type ${schemaName}.type_enum NOT NULL,
+                PRIMARY KEY (job_id, field),
+                FOREIGN KEY (job_id) REFERENCES ${fullTableName} (id) ON DELETE CASCADE
+              ) PARTITION BY HASH (job_id);
+            `);
+                        // Create partitions for attributes table
+                        await client.query(`
+              DO $$
+              BEGIN
+                FOR i IN 0..7 LOOP
+                  EXECUTE format(
+                    'CREATE TABLE IF NOT EXISTS ${attributesTableName}_part_%s PARTITION OF ${attributesTableName} 
+                    FOR VALUES WITH (modulus 8, remainder %s)',
+                    i, i
+                  );
+                END LOOP;
+              END$$;
+            `);
+                        // Create indexes for attributes table
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_attributes_type_field 
+              ON ${attributesTableName} (type, field);
+            `);
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_attributes_field 
+              ON ${attributesTableName} (field);
+            `);
+                        break;
+                    case 'list':
+                        await client.query(`
+              CREATE TABLE IF NOT EXISTS ${fullTableName} (
+                key TEXT NOT NULL,
+                index BIGINT NOT NULL,
+                value TEXT,
+                expiry TIMESTAMP WITH TIME ZONE,
+                PRIMARY KEY (key, index)
+              );
+            `);
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_expiry 
+              ON ${fullTableName} (key, expiry);
+            `);
+                        break;
+                    case 'sorted_set':
+                        await client.query(`
+              CREATE TABLE IF NOT EXISTS ${fullTableName} (
+                key TEXT NOT NULL,
+                member TEXT NOT NULL,
+                score DOUBLE PRECISION NOT NULL,
+                expiry TIMESTAMP WITH TIME ZONE,
+                PRIMARY KEY (key, member)
+              );
+            `);
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_key_score_member 
+              ON ${fullTableName} (key, score, member);
+            `);
+                        break;
+                    default:
+                        context.logger.warn(`Unknown table type for ${tableDef.name}`);
+                        break;
+                }
+            }
+            // Commit transaction
+            await client.query('COMMIT');
+        }
+        catch (error) {
+            context.logger.error('postgres-create-tables-error', { error });
+            await client.query('ROLLBACK');
+            throw error;
+        }
+    },
+    getTableNames(appName) {
+        const tableNames = [];
+        // Applications table (only hotmesh prefix)
+        tableNames.push('hotmesh_applications', 'hotmesh_connections');
+        // Other tables with appName
+        const tablesWithAppName = [
+            'throttles',
+            'roles',
+            'task_priorities',
+            'task_schedules',
+            'task_lists',
+            'events',
+            'jobs',
+            'stats_counted',
+            'stats_indexed',
+            'stats_ordered',
+            'versions',
+            'signal_patterns',
+            'signal_registry',
+            'symbols',
+        ];
+        tablesWithAppName.forEach((table) => {
+            tableNames.push(`${context.storeClient.safeName(appName)}.${table}`);
+        });
+        return tableNames;
+    },
+    getTableDefinitions(appName) {
+        const schemaName = context.storeClient.safeName(appName);
+        const tableDefinitions = [
+            {
+                schema: 'public',
+                name: 'hotmesh_applications',
+                type: 'hash',
+            },
+            {
+                schema: 'public',
+                name: 'hotmesh_connections',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'throttles',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'roles',
+                type: 'string',
+            },
+            {
+                schema: schemaName,
+                name: 'task_schedules',
+                type: 'sorted_set',
+            },
+            {
+                schema: schemaName,
+                name: 'task_priorities',
+                type: 'sorted_set',
+            },
+            {
+                schema: schemaName,
+                name: 'task_lists',
+                type: 'list',
+            },
+            {
+                schema: schemaName,
+                name: 'events',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'jobs',
+                type: 'jobhash', // Adds partitioning, indexes, and enum type
+            },
+            {
+                schema: schemaName,
+                name: 'stats_counted',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'stats_ordered',
+                type: 'sorted_set',
+            },
+            {
+                schema: schemaName,
+                name: 'stats_indexed',
+                type: 'list',
+            },
+            {
+                schema: schemaName,
+                name: 'versions',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'signal_patterns',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'symbols',
+                type: 'hash',
+            },
+            {
+                schema: schemaName,
+                name: 'signal_registry',
+                type: 'string',
+            },
+        ];
+        return tableDefinitions;
+    },
+});
+exports.KVTables = KVTables;

package/build/services/store/providers/postgres/kvtransaction.d.ts

@@ -0,0 +1,36 @@
+import { HSetOptions, KVSQLProviderTransaction, SetOptions, ZAddOptions } from '../../../../types/provider';
+import type { KVSQL } from './kvsql';
+export declare class KVTransaction implements KVSQLProviderTransaction {
+    private kvsql;
+    [key: string]: any;
+    private commands;
+    constructor(kvsql: KVSQL);
+    addCommand(sql: string, params: any[], returnType: string, transform?: (rows: any[]) => any): this;
+    set(key: string, value: string, options?: SetOptions): this;
+    setnx(key: string, value: string): this;
+    setnxex(key: string, value: string, expireSeconds: number): this;
+    get(key: string): this;
+    del(key: string): this;
+    expire(key: string, seconds: number): this;
+    hset(key: string, fields: Record<string, string>, options?: HSetOptions): this;
+    hget(key: string, field: string): this;
+    hdel(key: string, fields: string[]): this;
+    hmget(key: string, fields: string[]): this;
+    hgetall(key: string): this;
+    hincrbyfloat(key: string, field: string, increment: number): this;
+    hscan(key: string, cursor: string, count?: number): this;
+    lrange(key: string, start: number, end: number): this;
+    rpush(key: string, value: string | string[]): this;
+    lpush(key: string, value: string | string[]): this;
+    lpop(key: string): this;
+    lmove(source: string, destination: string, srcPosition: 'LEFT' | 'RIGHT', destPosition: 'LEFT' | 'RIGHT'): this;
+    zadd(key: string, score: number, member: string, options?: ZAddOptions): this;
+    zrange(key: string, start: number, stop: number): this;
+    zrangebyscore(key: string, min: number, max: number): this;
+    zrangebyscore_withscores(key: string, min: number, max: number): this;
+    zrem(key: string, member: string): this;
+    zrank(key: string, member: string): this;
+    scan(cursor: number, count?: number): this;
+    rename(oldKey: string, newKey: string): this;
+    exec(): Promise<any[]>;
+}