@hotmeshio/hotmesh 0.5.6 → 0.5.7

package/build/services/memflow/search.js

@@ -33,10 +33,6 @@ class Search {
          * @private
          */
         this.searchSessionIndex = 0;
-        /**
-         * @private
-         */
-        this.cachedFields = {};
         const keyParams = {
             appId: hotMeshClient.appId,
             jobId: workflowId,
@@ -111,6 +107,46 @@ class Search {
             }
         }
     }
+    /**
+     * Returns all user-defined attributes (udata) for a workflow.
+     * These are fields that start with underscore (_) and have type='udata'.
+     *
+     * @example
+     * ```typescript
+     * const allUserData = await Search.findAllUserData('job123', hotMeshClient);
+     * // Returns: { _status: "active", _counter: "42", _name: "test" }
+     * ```
+     */
+    static async findAllUserData(jobId, hotMeshClient) {
+        const keyParams = {
+            appId: hotMeshClient.appId,
+            jobId: jobId,
+        };
+        const key = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
+        const search = hotMeshClient.engine.search;
+        const rawResult = await search.updateContext(key, {
+            '@udata:all': ''
+        });
+        // Transform the result:
+        // 1. Remove underscore prefix from keys
+        // 2. Handle special fields (like exponential values)
+        const result = {};
+        for (const [key, value] of Object.entries(rawResult)) {
+            // Remove underscore prefix
+            const cleanKey = key.startsWith('_') ? key.slice(1) : key;
+            // Special handling for fields that use logarithmic storage
+            if (cleanKey === 'multer') {
+                // Convert from log value back to actual value
+                const expValue = Math.exp(Number(value));
+                // Round to nearest integer since log multiplication doesn't need decimal precision
+                result[cleanKey] = Math.round(expValue).toString();
+            }
+            else {
+                result[cleanKey] = value;
+            }
+        }
+        return result;
+    }
     /**
      * Returns an array of search indexes ids
      *
@@ -153,27 +189,31 @@ class Search {
         if (ssGuid in replay) {
             return Number(replay[ssGuid]);
         }
-        const fields = {};
+        // Prepare fields to set with udata format
+        let udataFields;
         if (typeof args[0] === 'object') {
+            // Object format: { field1: 'value1', field2: 'value2' }
+            udataFields = {};
             for (const [key, value] of Object.entries(args[0])) {
-                delete this.cachedFields[key];
-                fields[this.safeKey(key)] = value.toString();
+                udataFields[this.safeKey(key)] = value.toString();
             }
         }
         else {
+            // Array format: ['field1', 'value1', 'field2', 'value2']
+            udataFields = [];
             for (let i = 0; i < args.length; i += 2) {
                 const keyName = args[i];
-                delete this.cachedFields[keyName];
                 const key = this.safeKey(keyName);
                 const value = args[i + 1].toString();
-                fields[key] = value;
+                udataFields.push(key, value);
             }
         }
-        const fieldCount = await this.search.setFields(this.jobId, fields);
-        await this.search.setFields(this.jobId, {
-            [ssGuid]: fieldCount.toString(),
+        // Use single transactional call to update fields and store replay value
+        const result = await this.search.updateContext(this.jobId, {
+            '@udata:set': JSON.stringify(udataFields),
+            [ssGuid]: '', // Pass replay ID to hash module for transactional replay storage
         });
-        return fieldCount;
+        return result;
     }
     /**
      * Returns the value of the record data field, given a field id
@@ -183,13 +223,20 @@ class Search {
      * const value = await search.get('field1');
      */
     async get(id) {
+        const ssGuid = this.getSearchSessionGuid();
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            // Replay cache stores the field value
+            return replay[ssGuid];
+        }
         try {
-            if (id in this.cachedFields) {
-                return this.cachedFields[id];
-            }
-            const value = await this.search.getField(this.jobId, this.safeKey(id));
-            this.cachedFields[id] = value;
-            return value;
+            // Use server-side udata get operation with replay storage
+            const result = await this.search.updateContext(this.jobId, {
+                '@udata:get': this.safeKey(id),
+                [ssGuid]: '', // Pass replay ID to hash module
+            });
+            return result || '';
         }
         catch (error) {
             this.hotMeshClient.logger.error('memflow-search-get-error', {
@@ -202,29 +249,22 @@ class Search {
      * Returns the values of all specified fields in the HASH stored at key.
      */
     async mget(...args) {
-        let isCached = true;
-        const values = [];
-        const safeArgs = [];
-        for (let i = 0; i < args.length; i++) {
-            if (isCached && args[i] in this.cachedFields) {
-                values.push(this.cachedFields[args[i]]);
-            }
-            else {
-                isCached = false;
-            }
-            safeArgs.push(this.safeKey(args[i]));
+        const ssGuid = this.getSearchSessionGuid();
+        const store = storage_1.asyncLocalStorage.getStore();
+        const replay = store?.get('replay') ?? {};
+        if (ssGuid in replay) {
+            // Replay cache stores the field values array
+            const replayValue = replay[ssGuid];
+            return typeof replayValue === 'string' ? replayValue.split('|||') : replayValue;
         }
         try {
-            if (isCached) {
-                return values;
-            }
-            const returnValues = await this.search.getFields(this.jobId, safeArgs);
-            returnValues.forEach((value, index) => {
-                if (value !== null) {
-                    this.cachedFields[args[index]] = value;
-                }
+            const safeArgs = args.map(arg => this.safeKey(arg));
+            // Use server-side udata mget operation with replay storage
+            const result = await this.search.updateContext(this.jobId, {
+                '@udata:mget': JSON.stringify(safeArgs),
+                [ssGuid]: '', // Pass replay ID to hash module
             });
-            return returnValues;
+            return result || [];
         }
         catch (error) {
             this.hotMeshClient.logger.error('memflow-search-mget-error', {
@@ -245,23 +285,16 @@ class Search {
         const ssGuid = this.getSearchSessionGuid();
         const store = storage_1.asyncLocalStorage.getStore();
         const replay = store?.get('replay') ?? {};
-        const safeArgs = [];
-        for (let i = 0; i < args.length; i++) {
-            const keyName = args[i];
-            delete this.cachedFields[keyName];
-            safeArgs.push(this.safeKey(keyName));
-        }
         if (ssGuid in replay) {
             return Number(replay[ssGuid]);
         }
-        const response = await this.search.deleteFields(this.jobId, safeArgs);
-        const formattedResponse = isNaN(response)
-            ? 0
-            : Number(response);
-        await this.search.setFields(this.jobId, {
-            [ssGuid]: formattedResponse.toString(),
+        const safeArgs = args.map(arg => this.safeKey(arg));
+        // Use server-side udata delete operation with replay storage
+        const result = await this.search.updateContext(this.jobId, {
+            '@udata:delete': JSON.stringify(safeArgs),
+            [ssGuid]: '', // Pass replay ID to hash module for transactional replay storage
         });
-        return formattedResponse;
+        return Number(result || 0);
     }
     /**
      * Increments the value of a float field by the given amount. Returns the
@@ -273,16 +306,18 @@ class Search {
      * const count = await search.incr('field1', 1.5);
      */
     async incr(key, val) {
-        delete this.cachedFields[key];
         const ssGuid = this.getSearchSessionGuid();
         const store = storage_1.asyncLocalStorage.getStore();
         const replay = store?.get('replay') ?? {};
         if (ssGuid in replay) {
             return Number(replay[ssGuid]);
         }
-        const num = await this.search.incrementFieldByFloat(this.jobId, this.safeKey(key), val);
-        await this.search.setFields(this.jobId, { [ssGuid]: num.toString() });
-        return num;
+        // Use server-side udata increment operation with replay storage
+        const result = await this.search.updateContext(this.jobId, {
+            '@udata:increment': JSON.stringify({ field: this.safeKey(key), value: val }),
+            [ssGuid]: '', // Pass replay ID to hash module for transactional replay storage
+        });
+        return Number(result);
     }
     /**
      * Multiplies the value of a field by the given amount. Returns the
@@ -294,27 +329,19 @@ class Search {
      * const product = await search.mult('field1', 1.5);
      */
     async mult(key, val) {
-        delete this.cachedFields[key];
         const ssGuid = this.getSearchSessionGuid();
         const store = storage_1.asyncLocalStorage.getStore();
         const replay = store?.get('replay') ?? {};
         if (ssGuid in replay) {
             return Math.exp(Number(replay[ssGuid]));
         }
-        const ssGuidValue = await this.search.incrementFieldByFloat(this.jobId, ssGuid, 1);
-        if (ssGuidValue === 1) {
-            const log = Math.log(val);
-            const logTotal = await this.search.incrementFieldByFloat(this.jobId, this.safeKey(key), log);
-            await this.search.setFields(this.jobId, {
-                [ssGuid]: logTotal.toString(),
-            });
-            return Math.exp(logTotal);
-        }
-        else {
-            const logTotalStr = await this.search.getField(this.jobId, ssGuid);
-            const logTotal = Number(logTotalStr);
-            return Math.exp(logTotal);
-        }
+        // Use server-side udata multiply operation with replay storage
+        const result = await this.search.updateContext(this.jobId, {
+            '@udata:multiply': JSON.stringify({ field: this.safeKey(key), value: val }),
+            [ssGuid]: '', // Pass replay ID to hash module for transactional replay storage
+        });
+        // The result is the log value, so we need to exponentiate it
+        return Math.exp(Number(result));
     }
 }
 exports.Search = Search;

package/build/services/memflow/worker.d.ts

@@ -2,7 +2,7 @@ import { HotMesh } from '../hotmesh';
 import { Connection, Registry, WorkerConfig, WorkerOptions } from '../../types/memflow';
 /**
  * The *Worker* service Registers worker functions and connects them to the mesh,
- * using the target backend provider/s (Redis, Postgres, NATS, etc).
+ * using the target backend provider/s (Postgres, NATS, etc).
  *
  * @example
  * ```typescript

package/build/services/memflow/worker.js

@@ -13,7 +13,7 @@ const factory_1 = require("./schemas/factory");
 const index_1 = require("./index");
 /**
  * The *Worker* service Registers worker functions and connects them to the mesh,
- * using the target backend provider/s (Redis, Postgres, NATS, etc).
+ * using the target backend provider/s (Postgres, NATS, etc).
  *
  * @example
  * ```typescript

package/build/services/meshcall/index.d.ts

@@ -4,7 +4,7 @@ import { ProviderConfig, ProvidersConfig } from '../../types/provider';
 /**
  * MeshCall connects any function as an idempotent endpoint.
  * Call functions from anywhere on the network connected to the
- * target backend (Postgres, Redis/ValKey, NATS, etc). Function
+ * target backend (Postgres, NATS, etc). Function
  * responses are cacheable and invocations can be scheduled to
  * run as idempotent cron jobs (this one runs nightly at midnight
  * and uses Postgres as the backend provider).

package/build/services/meshcall/index.js

@@ -11,7 +11,7 @@ const factory_1 = require("./schemas/factory");
 /**
  * MeshCall connects any function as an idempotent endpoint.
  * Call functions from anywhere on the network connected to the
- * target backend (Postgres, Redis/ValKey, NATS, etc). Function
+ * target backend (Postgres, NATS, etc). Function
  * responses are cacheable and invocations can be scheduled to
  * run as idempotent cron jobs (this one runs nightly at midnight
  * and uses Postgres as the backend provider).

package/build/services/reporter/index.d.ts

@@ -15,7 +15,7 @@ declare class ReporterService {
     private validateOptions;
     private generateDateTimeSets;
     private convertRangeToMinutes;
-    private buildRedisKey;
+    private buildKeys;
     private aggregateData;
     private buildStatsResponse;
     private handleSegments;

package/build/services/reporter/index.js

@@ -13,10 +13,10 @@ class ReporterService {
         const { key, granularity, range, end, start } = options;
         this.validateOptions(options);
         const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
-        const redisKeys = dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime));
-        const rawData = await this.store.getJobStats(redisKeys);
+        const keys = dateTimeSets.map((dateTime) => this.buildKeys(key, dateTime));
+        const rawData = await this.store.getJobStats(keys);
         const [count, aggregatedData] = this.aggregateData(rawData);
-        const statsResponse = this.buildStatsResponse(rawData, redisKeys, aggregatedData, count, options);
+        const statsResponse = this.buildStatsResponse(rawData, keys, aggregatedData, count, options);
         return statsResponse;
     }
     validateOptions(options) {
@@ -93,7 +93,7 @@ class ReporterService {
                 return null;
         }
     }
-    buildRedisKey(key, dateTime, subTarget = '') {
+    buildKeys(key, dateTime, subTarget = '') {
         return `hmsh:${this.appVersion.id}:s:${key}:${dateTime}${subTarget ? ':' + subTarget : ''}`;
     }
     aggregateData(rawData) {
@@ -115,12 +115,12 @@ class ReporterService {
         });
         return [count, aggregatedData];
     }
-    buildStatsResponse(rawData, redisKeys, aggregatedData, count, options) {
+    buildStatsResponse(rawData, keys, aggregatedData, count, options) {
         const measures = [];
         const measureKeys = Object.keys(aggregatedData).filter((key) => key !== 'count');
         let segments = undefined;
         if (options.sparse !== true) {
-            segments = this.handleSegments(rawData, redisKeys);
+            segments = this.handleSegments(rawData, keys);
         }
         measureKeys.forEach((key) => {
             const measure = {
@@ -179,12 +179,12 @@ class ReporterService {
         }
         const { key, granularity, range, end, start } = options;
         this.validateOptions(options);
-        let redisKeys = [];
+        let keys = [];
         facets.forEach((facet) => {
             const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
-            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
+            keys = keys.concat(dateTimeSets.map((dateTime) => this.buildKeys(key, dateTime, `index:${facet}`)));
         });
-        const idsData = await this.store.getJobIds(redisKeys, idRange);
+        const idsData = await this.store.getJobIds(keys, idRange);
         const idsResponse = this.buildIdsResponse(idsData, options, facets);
         return idsResponse;
     }
@@ -254,12 +254,12 @@ class ReporterService {
         }
         const { key, granularity, range, end, start } = options;
         this.validateOptions(options);
-        let redisKeys = [];
+        let keys = [];
         facets.forEach((facet) => {
             const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
-            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
+            keys = keys.concat(dateTimeSets.map((dateTime) => this.buildKeys(key, dateTime, `index:${facet}`)));
         });
-        const idsData = await this.store.getJobIds(redisKeys, [0, 1]);
+        const idsData = await this.store.getJobIds(keys, [0, 1]);
         const workerLists = this.buildWorkerLists(idsData);
         return workerLists;
     }

package/build/services/search/factory.js

@@ -3,20 +3,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SearchServiceFactory = void 0;
 const utils_1 = require("../../modules/utils");
 const postgres_1 = require("./providers/postgres/postgres");
-const ioredis_1 = require("./providers/redis/ioredis");
-const redis_1 = require("./providers/redis/redis");
 class SearchServiceFactory {
     static async init(providerClient, storeProviderClient, namespace, appId, logger) {
         let service;
         if ((0, utils_1.identifyProvider)(providerClient) === 'postgres') {
             service = new postgres_1.PostgresSearchService(providerClient, storeProviderClient);
         }
-        else if ((0, utils_1.identifyProvider)(providerClient) === 'redis') {
-            service = new redis_1.RedisSearchService(providerClient, storeProviderClient);
-        }
-        else {
-            service = new ioredis_1.IORedisSearchService(providerClient, storeProviderClient);
-        }
         await service.init(namespace, appId, logger);
         return service;
     }

package/build/services/search/providers/postgres/postgres.js

@@ -11,7 +11,7 @@ class PostgresSearchService extends index_1.SearchService {
     constructor(searchClient, storeClient) {
         super(searchClient, storeClient);
         this.pgClient = searchClient; //raw pg client (to send raw sql)
-        this.searchClient = new kvsql_1.KVSQL(//wrapped pg client (to send as redis commands)
+        this.searchClient = new kvsql_1.KVSQL(//wrapped pg client
         searchClient, this.namespace, this.appId);
     }
     async init(namespace, appId, logger) {

package/build/services/store/cache.d.ts

@@ -1,5 +1,5 @@
 /**
- * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * The Cache is a key/value store and used to store commonly accessed metadata
  * (mainly the execution rules for the app) to save time accessing them as they
  * are immutable per verison. Rules are only ejected when a new version
  * (a new distributed executable) is deployed to the quorum

package/build/services/store/cache.js

@@ -1,6 +1,6 @@
 "use strict";
 /**
- * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * The Cache is a key/value store and used to store commonly accessed metadata
  * (mainly the execution rules for the app) to save time accessing them as they
  * are immutable per verison. Rules are only ejected when a new version
  * (a new distributed executable) is deployed to the quorum

package/build/services/store/factory.js

@@ -2,19 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StoreServiceFactory = void 0;
 const utils_1 = require("../../modules/utils");
-const ioredis_1 = require("./providers/redis/ioredis");
-const redis_1 = require("./providers/redis/redis");
 const postgres_1 = require("./providers/postgres/postgres");
 class StoreServiceFactory {
     static async init(providerClient, namespace, appId, logger) {
         let service;
-        if ((0, utils_1.identifyProvider)(providerClient) === 'redis') {
-            service = new redis_1.RedisStoreService(providerClient);
-        }
-        else if ((0, utils_1.identifyProvider)(providerClient) === 'ioredis') {
-            service = new ioredis_1.IORedisStoreService(providerClient);
-        }
-        else if ((0, utils_1.identifyProvider)(providerClient) === 'postgres') {
+        if ((0, utils_1.identifyProvider)(providerClient) === 'postgres') {
             service = new postgres_1.PostgresStoreService(providerClient);
         } //etc
         await service.init(namespace, appId, logger);

package/build/services/store/index.d.ts

@@ -65,7 +65,7 @@ declare abstract class StoreService<Provider extends ProviderClient, Transaction
     abstract getActiveTaskQueue(): Promise<string | null>;
     abstract deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
     abstract processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
-    abstract expireJob(jobId: string, inSeconds: number, redisMulti?: TransactionProvider): Promise<void>;
+    abstract expireJob(jobId: string, inSeconds: number, txProvider?: TransactionProvider): Promise<void>;
     abstract getDependencies(jobId: string): Promise<string[]>;
     abstract delistSignalKey(key: string, target: string): Promise<void>;
     abstract registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, transaction?: TransactionProvider): Promise<void>;

package/build/services/store/providers/postgres/kvtypes/hash/index.js

@@ -17,12 +17,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.deriveType = exports.isJobsTable = exports.hashModule = void 0;
 const basic_1 = require("./basic");
 const jsonb_1 = require("./jsonb");
+const udata_1 = require("./udata");
 const scan_1 = require("./scan");
 const expire_1 = require("./expire");
 const utils_1 = require("./utils");
 const hashModule = (context) => {
     const basicOps = (0, basic_1.createBasicOperations)(context);
     const jsonbOps = (0, jsonb_1.createJsonbOperations)(context);
+    const udataOps = (0, udata_1.createUdataOperations)(context);
     const scanOps = (0, scan_1.createScanOperations)(context);
     const expireOps = (0, expire_1.createExpireOperations)(context);
     return {
@@ -84,6 +86,37 @@ const hashModule = (context) => {
                     return executeJsonbOperation(sql, params, multi);
                 }
             }
+            // Handle udata operations for search fields
+            if (isJobsTableResult) {
+                if ('@udata:set' in fields) {
+                    const { sql, params } = udataOps.handleUdataSet(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+                else if ('@udata:get' in fields) {
+                    const { sql, params } = udataOps.handleUdataGet(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+                else if ('@udata:mget' in fields) {
+                    const { sql, params } = udataOps.handleUdataMget(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+                else if ('@udata:delete' in fields) {
+                    const { sql, params } = udataOps.handleUdataDelete(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+                else if ('@udata:increment' in fields) {
+                    const { sql, params } = udataOps.handleUdataIncrement(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+                else if ('@udata:multiply' in fields) {
+                    const { sql, params } = udataOps.handleUdataMultiply(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+                else if ('@udata:all' in fields) {
+                    const { sql, params } = udataOps.handleUdataAll(key, fields, options);
+                    return executeJsonbOperation(sql, params, multi);
+                }
+            }
             // Fall back to basic hset for all other cases
             return basicOps.hset(key, fields, options, multi);
         },
@@ -127,6 +160,30 @@ const hashModule = (context) => {
                     return jsonbOps.handleContextGet(key, fields, options);
                 }
             }
+            // Handle udata operations for search fields
+            if (isJobsTableResult) {
+                if ('@udata:set' in fields) {
+                    return udataOps.handleUdataSet(key, fields, options);
+                }
+                else if ('@udata:get' in fields) {
+                    return udataOps.handleUdataGet(key, fields, options);
+                }
+                else if ('@udata:mget' in fields) {
+                    return udataOps.handleUdataMget(key, fields, options);
+                }
+                else if ('@udata:delete' in fields) {
+                    return udataOps.handleUdataDelete(key, fields, options);
+                }
+                else if ('@udata:increment' in fields) {
+                    return udataOps.handleUdataIncrement(key, fields, options);
+                }
+                else if ('@udata:multiply' in fields) {
+                    return udataOps.handleUdataMultiply(key, fields, options);
+                }
+                else if ('@udata:all' in fields) {
+                    return udataOps.handleUdataAll(key, fields, options);
+                }
+            }
             // Use the imported _hset function
             return (0, basic_1._hset)(context, key, fields, options);
         },

package/build/services/store/providers/postgres/kvtypes/hash/udata.d.ts

@@ -0,0 +1,10 @@
+import { HashContext, SqlResult, HSetOptions } from './types';
+export declare function createUdataOperations(context: HashContext['context']): {
+    handleUdataSet: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+    handleUdataGet: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+    handleUdataMget: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+    handleUdataDelete: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+    handleUdataIncrement: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+    handleUdataMultiply: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+    handleUdataAll: (key: string, fields: Record<string, string>, options?: HSetOptions) => SqlResult;
+};