@hotmeshio/hotmesh 0.4.0 → 0.4.2

package/build/services/store/index.d.ts

@@ -0,0 +1,89 @@
+import { KeyStoreParams, KeyType } from '../../modules/key';
+import { ILogger } from '../logger';
+import { SerializerService as Serializer } from '../serializer';
+import { Consumes } from '../../types/activity';
+import { AppVID } from '../../types/app';
+import { HookRule, HookSignal } from '../../types/hook';
+import { HotMeshApps, HotMeshSettings } from '../../types/hotmesh';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { ThrottleOptions } from '../../types/quorum';
+import { StringAnyType, Symbols, StringStringType, SymbolSets } from '../../types/serializer';
+import { IdsData, JobStatsRange, StatsType } from '../../types/stats';
+import { WorkListTaskType } from '../../types/task';
+import { Cache } from './cache';
+declare abstract class StoreService<Provider extends ProviderClient, TransactionProvider extends ProviderTransaction> {
+    storeClient: Provider;
+    namespace: string;
+    appId: string;
+    logger: ILogger;
+    cache: Cache;
+    serializer: Serializer;
+    constructor(client: Provider);
+    abstract transact(): TransactionProvider;
+    abstract init(namespace: string, appId: string, logger: ILogger): Promise<HotMeshApps>;
+    abstract mintKey(type: KeyType, params: KeyStoreParams): string;
+    abstract getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
+    abstract setSettings(manifest: HotMeshSettings): Promise<any>;
+    abstract getApp(id: string, refresh?: boolean): Promise<any>;
+    abstract setApp(id: string, version: string): Promise<any>;
+    abstract activateAppVersion(id: string, version: string): Promise<boolean>;
+    abstract reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay?: number): Promise<boolean>;
+    abstract releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean>;
+    abstract reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY', tryCount?: number): Promise<[number, number, Symbols]>;
+    abstract getSymbols(activityId: string): Promise<Symbols>;
+    abstract addSymbols(activityId: string, symbols: Symbols): Promise<boolean>;
+    abstract getSymbolValues(): Promise<Symbols>;
+    abstract addSymbolValues(symvals: Symbols): Promise<boolean>;
+    abstract getSymbolKeys(symbolNames: string[]): Promise<SymbolSets>;
+    abstract setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, transaction?: TransactionProvider): Promise<any>;
+    abstract getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
+    abstract getJobIds(indexKeys: string[], idRange: [number, number]): Promise<IdsData>;
+    abstract setStatus(collationKeyStatus: number, jobId: string, appId: string, transaction?: TransactionProvider): Promise<any>;
+    abstract getStatus(jobId: string, appId: string): Promise<number>;
+    abstract setStateNX(jobId: string, appId: string, status?: number): Promise<boolean>;
+    abstract setState(state: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, transaction?: TransactionProvider): Promise<string>;
+    abstract getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
+    abstract getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
+    abstract getRaw(jobId: string): Promise<StringStringType>;
+    abstract collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, transaction?: TransactionProvider): Promise<number>;
+    abstract collateSynthetic(jobId: string, guid: string, amount: number, transaction?: TransactionProvider): Promise<number>;
+    abstract getSchema(activityId: string, appVersion: AppVID): Promise<any>;
+    abstract getSchemas(appVersion: AppVID): Promise<Record<string, any>>;
+    abstract setSchemas(schemas: Record<string, any>, appVersion: AppVID): Promise<any>;
+    abstract setSubscriptions(subscriptions: Record<string, any>, appVersion: AppVID): Promise<boolean>;
+    abstract getSubscriptions(appVersion: AppVID): Promise<Record<string, any>>;
+    abstract getSubscription(topic: string, appVersion: AppVID): Promise<string | undefined>;
+    abstract setTransitions(transitions: Record<string, any>, appVersion: AppVID): Promise<any>;
+    abstract getTransitions(appVersion: AppVID): Promise<any>;
+    abstract setHookRules(hookRules: Record<string, HookRule[]>): Promise<any>;
+    abstract getHookRules(): Promise<Record<string, HookRule[]>>;
+    abstract getAllSymbols(): Promise<Symbols>;
+    abstract setHookSignal(hook: HookSignal, transaction?: TransactionProvider): Promise<any>;
+    abstract getHookSignal(topic: string, resolved: string): Promise<string | undefined>;
+    abstract deleteHookSignal(topic: string, resolved: string): Promise<number | undefined>;
+    abstract addTaskQueues(keys: string[]): Promise<void>;
+    abstract getActiveTaskQueue(): Promise<string | null>;
+    abstract deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
+    abstract processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
+    abstract expireJob(jobId: string, inSeconds: number, redisMulti?: TransactionProvider): Promise<void>;
+    abstract getDependencies(jobId: string): Promise<string[]>;
+    abstract delistSignalKey(key: string, target: string): Promise<void>;
+    abstract registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, transaction?: TransactionProvider): Promise<void>;
+    abstract getNextTask(listKey?: string): Promise<[
+        listKey: string,
+        jobId: string,
+        gId: string,
+        activityId: string,
+        type: WorkListTaskType
+    ] | boolean>;
+    abstract interrupt(topic: string, jobId: string, options: {
+        [key: string]: any;
+    }): Promise<void>;
+    abstract scrub(jobId: string): Promise<void>;
+    abstract findJobs(queryString?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, string[]]>;
+    abstract findJobFields(jobId: string, fieldMatchPattern?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, StringStringType]>;
+    abstract setThrottleRate(options: ThrottleOptions): Promise<void>;
+    abstract getThrottleRates(): Promise<StringStringType>;
+    abstract getThrottleRate(topic: string): Promise<number>;
+}
+export { StoreService };

package/build/services/store/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StoreService = void 0;
+class StoreService {
+    constructor(client) {
+        this.storeClient = client;
+    }
+}
+exports.StoreService = StoreService;

package/build/services/store/providers/postgres/kvsql.d.ts

@@ -0,0 +1,168 @@
+import { KeyStoreParams } from '../../../../types/hotmesh';
+import { PostgresClientType } from '../../../../types/postgres';
+import { ProviderTransaction } from '../../../../types/provider';
+import { stringModule } from './kvtypes/string';
+import { hashModule } from './kvtypes/hash';
+import { listModule } from './kvtypes/list';
+import { zsetModule } from './kvtypes/zset';
+/**
+ * KVSQL is a class that provides a set of methods to interact with a Postgres database.
+ * It is used to interact with the database in a key-value manner.
+ */
+export declare class KVSQL {
+    pgClient: PostgresClientType;
+    namespace: string;
+    appId: string;
+    string: ReturnType<typeof stringModule>;
+    hash: ReturnType<typeof hashModule>;
+    list: ReturnType<typeof listModule>;
+    zset: ReturnType<typeof zsetModule>;
+    constructor(pgClient: PostgresClientType, namespace: string, appId: string);
+    isStatusOnly(fields: string[]): boolean;
+    appendExpiryClause(baseQuery: string, tableAlias: string): string;
+    appendJobExpiryClause(baseQuery: string, tableAlias: string): string;
+    getMulti(): ProviderTransaction;
+    transact(): ProviderTransaction;
+    exec(...args: any[]): Promise<Array<any>>;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    /**
+     * Resolves the table name when provided a key
+     */
+    tableForKey(key: string, stats_type?: 'hash' | 'sorted_set' | 'list'): string;
+    safeName(input: string, prefix?: string): string;
+    set: (key: string, value: string, options?: import("../../../../types/provider").SetOptions, multi?: ProviderTransaction) => Promise<boolean>;
+    _set: (key: string, value: string, options?: import("../../../../types/provider").SetOptions) => {
+        sql: string;
+        params: any[];
+    };
+    get: (key: string, multi?: ProviderTransaction) => Promise<string>;
+    _get: (key: string) => {
+        sql: string;
+        params: any[];
+    };
+    del: (key: string, multi?: ProviderTransaction) => Promise<number>;
+    _del: (key: string) => {
+        sql: string;
+        params: any[];
+    };
+    setnx: (key: string, value: string, multi?: ProviderTransaction) => Promise<boolean>;
+    setnxex: (key: string, value: string, delay: number, multi?: ProviderTransaction) => Promise<boolean>;
+    hset: (key: string, fields: Record<string, string>, options?: import("../../../../types/provider").HSetOptions, multi?: ProviderTransaction) => Promise<any>;
+    _hset: (key: string, fields: Record<string, string>, options?: import("../../../../types/provider").HSetOptions) => {
+        sql: string;
+        params: any[];
+    };
+    hsetnx: (key: string, field: string, value: string, multi?: ProviderTransaction) => Promise<number>;
+    hget: (key: string, field: string, multi?: ProviderTransaction) => Promise<string>;
+    _hget: (key: string, field: string) => {
+        sql: string;
+        params: any[];
+    };
+    hdel: (key: string, fields: string[], multi?: unknown) => Promise<number>;
+    _hdel: (key: string, fields: string[]) => {
+        sql: string;
+        params: any[];
+    };
+    hmget: (key: string, fields: string[], multi?: ProviderTransaction) => Promise<string[]>;
+    _hmget: (key: string, fields: string[]) => {
+        sql: string;
+        params: any[];
+    };
+    hgetall: (key: string, multi?: ProviderTransaction) => Promise<Record<string, string>>;
+    hincrbyfloat: (key: string, field: string, increment: number, multi?: ProviderTransaction) => Promise<number>;
+    _hincrbyfloat: (key: string, field: string, increment: number) => {
+        sql: string;
+        params: any[];
+    };
+    hscan: (key: string, cursor: string, count?: number, pattern?: string, multi?: ProviderTransaction) => Promise<import("../../../../types/provider").HScanResult>;
+    _hscan: (key: string, cursor: string, count: number, pattern?: string) => {
+        sql: string;
+        params: any[];
+    };
+    expire: (key: string, seconds: number, multi?: ProviderTransaction) => Promise<boolean>;
+    _expire: (key: string, seconds: number) => {
+        sql: string;
+        params: any[];
+    };
+    scan: (cursor: number, count?: number, pattern?: string, multi?: ProviderTransaction) => Promise<{
+        cursor: number;
+        keys: string[];
+    }>;
+    _scan: (cursor: number, count: number, pattern?: string) => {
+        sql: string;
+        params: any[];
+    };
+    lrange: (key: string, start: number, end: number, multi?: ProviderTransaction) => Promise<string[]>;
+    _lrange: (key: string, start: number, end: number) => {
+        sql: string;
+        params: any[];
+    };
+    rpush: (key: string, value: string | string[], multi?: ProviderTransaction) => Promise<number>;
+    _rpush: (key: string, value: string | string[]) => {
+        sql: string;
+        params: any[];
+    };
+    lpush: (key: string, value: string | string[], multi?: ProviderTransaction) => Promise<number>;
+    _lpush: (key: string, value: string | string[]) => {
+        sql: string;
+        params: any[];
+    };
+    lpop: (key: string, multi?: ProviderTransaction) => Promise<string>;
+    _lpop: (key: string) => {
+        sql: string;
+        params: any[];
+    };
+    lmove: (source: string, destination: string, srcPosition: "LEFT" | "RIGHT", destPosition: "LEFT" | "RIGHT", multi?: ProviderTransaction) => Promise<string>;
+    _lmove: (source: string, destination: string, srcPosition: "LEFT" | "RIGHT", destPosition: "LEFT" | "RIGHT") => {
+        sql: string;
+        params: any[];
+    };
+    rename: (oldKey: string, newKey: string, multi?: ProviderTransaction) => Promise<void>;
+    _rename: (oldKey: string, newKey: string) => {
+        sql: string;
+        params: any[];
+    };
+    zadd: (key: string, score: number, member: string, options?: import("../../../../types/provider").ZAddOptions, multi?: ProviderTransaction) => Promise<number>;
+    _zadd: (key: string, score: number, member: string, options?: import("../../../../types/provider").ZAddOptions) => {
+        sql: string;
+        params: any[];
+    };
+    zrange: (key: string, start: number, stop: number, facet?: "WITHSCORES", multi?: ProviderTransaction) => Promise<string[]>;
+    _zrange: (key: string, start: number, stop: number, facet?: "WITHSCORES") => {
+        sql: string;
+        params: any[];
+    };
+    zrangebyscore: (key: string, min: number, max: number, multi?: ProviderTransaction) => Promise<string[]>;
+    _zrangebyscore: (key: string, min: number, max: number) => {
+        sql: string;
+        params: any[];
+    };
+    zrangebyscore_withscores: (key: string, min: number, max: number, multi?: ProviderTransaction) => Promise<{
+        member: string;
+        score: number;
+    }[]>;
+    _zrangebyscore_withscores: (key: string, min: number, max: number) => {
+        sql: string;
+        params: any[];
+    };
+    zrem: (key: string, member: string, multi?: ProviderTransaction) => Promise<number>;
+    _zrem: (key: string, member: string) => {
+        sql: string;
+        params: any[];
+    };
+    zrank: (key: string, member: string, multi?: ProviderTransaction) => Promise<number>;
+    _zrank: (key: string, member: string) => {
+        sql: string;
+        params: any[];
+    };
+    zscore: (key: string, member: string, multi?: ProviderTransaction) => Promise<number>;
+    _zscore: (key: string, member: string) => {
+        sql: string;
+        params: any[];
+    };
+    exists(key: string): Promise<string | 0>;
+    _exists(key: string): {
+        sql: string;
+        params: any[];
+    };
+}

package/build/services/store/providers/postgres/kvsql.js

@@ -0,0 +1,198 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KVSQL = void 0;
+const key_1 = require("../../../../modules/key");
+const kvtransaction_1 = require("./kvtransaction");
+const string_1 = require("./kvtypes/string");
+const hash_1 = require("./kvtypes/hash");
+const list_1 = require("./kvtypes/list");
+const zset_1 = require("./kvtypes/zset");
+/**
+ * KVSQL is a class that provides a set of methods to interact with a Postgres database.
+ * It is used to interact with the database in a key-value manner.
+ */
+class KVSQL {
+    constructor(pgClient, namespace, appId) {
+        // String Commands
+        this.set = (...args) => this.string.set(...args);
+        this._set = (...args) => this.string._set(...args);
+        this.get = (...args) => this.string.get(...args);
+        this._get = (...args) => this.string._get(...args);
+        this.del = (...args) => this.string.del(...args);
+        this._del = (...args) => this.string._del(...args);
+        this.setnx = (...args) => this.string.setnx(...args);
+        this.setnxex = (...args) => this.string.setnxex(...args);
+        // Hash Commands
+        this.hset = (...args) => this.hash.hset(...args);
+        this._hset = (...args) => this.hash._hset(...args);
+        this.hsetnx = (...args) => this.hash.hsetnx(...args);
+        this.hget = (...args) => this.hash.hget(...args);
+        this._hget = (...args) => this.hash._hget(...args);
+        this.hdel = (...args) => this.hash.hdel(...args);
+        this._hdel = (...args) => this.hash._hdel(...args);
+        this.hmget = (...args) => this.hash.hmget(...args);
+        this._hmget = (...args) => this.hash._hmget(...args);
+        this.hgetall = (...args) => this.hash.hgetall(...args);
+        this.hincrbyfloat = (...args) => this.hash.hincrbyfloat(...args);
+        this._hincrbyfloat = (...args) => this.hash._hincrbyfloat(...args);
+        this.hscan = (...args) => this.hash.hscan(...args);
+        this._hscan = (...args) => this.hash._hscan(...args);
+        this.expire = (...args) => this.hash.expire(...args);
+        this._expire = (...args) => this.hash._expire(...args);
+        this.scan = (...args) => this.hash.scan(...args);
+        this._scan = (...args) => this.hash._scan(...args);
+        // List Commands
+        this.lrange = (...args) => this.list.lrange(...args);
+        this._lrange = (...args) => this.list._lrange(...args);
+        this.rpush = (...args) => this.list.rpush(...args);
+        this._rpush = (...args) => this.list._rpush(...args);
+        this.lpush = (...args) => this.list.lpush(...args);
+        this._lpush = (...args) => this.list._lpush(...args);
+        this.lpop = (...args) => this.list.lpop(...args);
+        this._lpop = (...args) => this.list._lpop(...args);
+        this.lmove = (...args) => this.list.lmove(...args);
+        this._lmove = (...args) => this.list._lmove(...args);
+        this.rename = (...args) => this.list.rename(...args);
+        this._rename = (...args) => this.list._rename(...args);
+        // Sorted Set Commands
+        this.zadd = (...args) => this.zset.zadd(...args);
+        this._zadd = (...args) => this.zset._zadd(...args);
+        this.zrange = (...args) => this.zset.zrange(...args);
+        this._zrange = (...args) => this.zset._zrange(...args);
+        this.zrangebyscore = (...args) => this.zset.zrangebyscore(...args);
+        this._zrangebyscore = (...args) => this.zset._zrangebyscore(...args);
+        this.zrangebyscore_withscores = (...args) => this.zset.zrangebyscore_withscores(...args);
+        this._zrangebyscore_withscores = (...args) => this.zset._zrangebyscore_withscores(...args);
+        this.zrem = (...args) => this.zset.zrem(...args);
+        this._zrem = (...args) => this.zset._zrem(...args);
+        this.zrank = (...args) => this.zset.zrank(...args);
+        this._zrank = (...args) => this.zset._zrank(...args);
+        this.zscore = (...args) => this.zset.zscore(...args);
+        this._zscore = (...args) => this.zset._zscore(...args);
+        this.pgClient = pgClient;
+        this.namespace = namespace;
+        this.appId = appId;
+        this.hash = (0, hash_1.hashModule)(this);
+        this.list = (0, list_1.listModule)(this);
+        this.zset = (0, zset_1.zsetModule)(this);
+        this.string = (0, string_1.stringModule)(this);
+    }
+    isStatusOnly(fields) {
+        return fields.length === 1 && fields[0] === ':';
+    }
+    appendExpiryClause(baseQuery, tableAlias) {
+        return `
+      ${baseQuery}
+      AND (${tableAlias}.expiry IS NULL OR ${tableAlias}.expiry > NOW())
+    `;
+    }
+    appendJobExpiryClause(baseQuery, tableAlias) {
+        return `
+      ${baseQuery}
+      AND (${tableAlias}.expired_at IS NULL OR ${tableAlias}.expired_at > NOW())
+    `;
+    }
+    getMulti() {
+        return new kvtransaction_1.KVTransaction(this);
+    }
+    transact() {
+        return new kvtransaction_1.KVTransaction(this);
+    }
+    exec(...args) {
+        return Promise.resolve([]);
+    }
+    mintKey(type, params) {
+        //no-op
+        return '';
+    }
+    /**
+     * Resolves the table name when provided a key
+     */
+    tableForKey(key, stats_type) {
+        if (key === key_1.HMNS) {
+            return 'public.hotmesh_connections';
+        }
+        const [_, appName, abbrev, ...rest] = key.split(':');
+        if (appName === 'a') {
+            return 'public.hotmesh_applications';
+        }
+        const id = rest?.length ? rest.join(':') : '';
+        const entity = key_1.KeyService.resolveEntityType(abbrev, id);
+        if (this.safeName(this.appId) !== this.safeName(appName)) {
+            throw new Error(`App ID mismatch: ${this.appId} !== ${appName}`);
+        }
+        const schemaName = this.safeName(appName);
+        if (entity === 'stats') {
+            let tableName;
+            if (stats_type === 'sorted_set') {
+                tableName = 'stats_ordered';
+            }
+            else if (stats_type === 'list' || key.endsWith(':processed')) {
+                tableName = 'stats_indexed';
+            }
+            else if (stats_type === 'hash' || /:\d$/.test(key)) {
+                tableName = 'stats_counted';
+            }
+            else {
+                throw new Error(`Unknown stats type [${stats_type}] for key [${key}]`);
+            }
+            return `${schemaName}.${tableName}`;
+        }
+        if (entity === 'unknown_entity') {
+            throw new Error(`Unknown entity type abbreviation: ${abbrev}`);
+        }
+        else if (entity === 'applications') {
+            return 'public.hotmesh_applications';
+        }
+        else {
+            return `${schemaName}.${entity}`;
+        }
+    }
+    safeName(input, prefix = '') {
+        if (!input) {
+            return 'connections';
+        }
+        let tableName = input.trim().toLowerCase();
+        tableName = tableName.replace(/[^a-z0-9]+/g, '_');
+        if (prefix) {
+            tableName = `${prefix}_${tableName}`;
+        }
+        if (tableName.length > 63) {
+            tableName = tableName.slice(0, 63);
+        }
+        tableName = tableName.replace(/_+$/g, '');
+        if (!tableName) {
+            tableName = 'connections';
+        }
+        return tableName;
+    }
+    async exists(key) {
+        const { sql, params } = this._exists(key);
+        const res = await this.pgClient.query(sql, params);
+        return res.rows.length ? res.rows[0].table_name : 0;
+    }
+    _exists(key) {
+        const tableName = this.tableForKey(key);
+        const isJobsTable = tableName.endsWith('jobs');
+        let sql;
+        if (isJobsTable) {
+            sql = `
+        SELECT FROM ${tableName}
+        WHERE id = $1
+          AND (expired_at IS NULL OR expired_at > NOW())
+        LIMIT 1;
+      `;
+        }
+        else {
+            sql = `
+        SELECT FROM ${tableName}
+        WHERE key = $1
+          AND (expiry IS NULL OR expiry > NOW())
+        LIMIT 1;
+      `;
+        }
+        const params = [key];
+        return { sql, params };
+    }
+}
+exports.KVSQL = KVSQL;

package/build/services/store/providers/postgres/kvtables.d.ts

@@ -0,0 +1,20 @@
+import { PostgresClientType, PostgresPoolClientType } from '../../../../types/postgres';
+import type { PostgresStoreService } from './postgres';
+export declare const KVTables: (context: PostgresStoreService) => {
+    /**
+     * Deploys the necessary tables with the specified naming strategy.
+     * @param appName - The name of the application.
+     */
+    deploy(appName: string): Promise<void>;
+    getAdvisoryLockId(appName: string): number;
+    hashStringToInt(str: string): number;
+    waitForTablesCreation(lockId: number, appName: string): Promise<void>;
+    checkIfTablesExist(client: PostgresClientType, appName: string): Promise<boolean>;
+    createTables(client: PostgresClientType | PostgresPoolClientType, appName: string): Promise<void>;
+    getTableNames(appName: string): string[];
+    getTableDefinitions(appName: string): Array<{
+        schema: string;
+        name: string;
+        type: string;
+    }>;
+};