@hotmeshio/hotmesh 0.0.48 → 0.0.50

package/build/services/worker/index.js

@@ -12,6 +12,7 @@ const redis_2 = require("../stream/clients/redis");
 const ioredis_3 = require("../sub/clients/ioredis");
 const redis_3 = require("../sub/clients/redis");
 const stream_1 = require("../../types/stream");
+const enums_1 = require("../../modules/enums");
 class WorkerService {
     constructor() {
         this.reporting = false;
@@ -26,6 +27,7 @@ class WorkerService {
                 service.namespace = namespace;
                 service.appId = appId;
                 service.guid = guid;
+                service.callback = worker.callback;
                 service.topic = worker.topic;
                 service.config = config;
                 service.logger = logger;
@@ -95,14 +97,51 @@ class WorkerService {
         return async (topic, message) => {
             self.logger.debug('worker-event-received', { topic, type: message.type });
             if (message.type === 'throttle') {
-                self.throttle(message.throttle);
+                if (message.topic !== null) { //undefined allows passthrough
+                    self.throttle(message.throttle);
+                }
             }
             else if (message.type === 'ping') {
                 self.sayPong(self.appId, self.guid, message.originator, message.details);
             }
+            else if (message.type === 'rollcall') {
+                if (message.topic !== null) { //undefined allows passthrough
+                    self.doRollCall(message);
+                }
+            }
         };
     }
-    async sayPong(appId, guid, originator, details = false) {
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    async doRollCall(message) {
+        let iteration = 0;
+        let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
+        if (this.rollCallInterval)
+            clearTimeout(this.rollCallInterval);
+        const base = (message.interval / 2);
+        const amount = base + Math.ceil(Math.random() * base);
+        do {
+            await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
+            await this.sayPong(this.appId, this.guid, null, true, message.signature);
+            if (!message.interval)
+                return;
+            const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
+            this.rollCallInterval = timerId;
+            await promise;
+        } while (this.rollCallInterval && iteration++ < max - 1);
+    }
+    cancelRollCall() {
+        if (this.rollCallInterval) {
+            clearTimeout(this.rollCallInterval);
+            delete this.rollCallInterval;
+        }
+    }
+    stop() {
+        this.cancelRollCall();
+    }
+    async sayPong(appId, guid, originator, details = false, signature = false) {
         let profile;
         if (details) {
             const params = {
@@ -122,11 +161,13 @@ class WorkerService {
                 reclaimDelay: this.router.reclaimDelay,
                 reclaimCount: this.router.reclaimCount,
                 system: await (0, utils_1.getSystemHealth)(),
+                signature: signature ? this.callback.toString() : undefined,
             };
         }
         this.store.publish(key_1.KeyType.QUORUM, {
             type: 'pong',
-            guid, originator,
+            guid,
+            originator,
             profile,
         }, appId);
     }

package/build/types/activity.d.ts

@@ -74,6 +74,7 @@ interface AwaitActivity extends BaseActivity {
     type: 'await';
     eventName: string;
     timeout: number;
+    await?: boolean;
 }
 interface WorkerActivity extends BaseActivity {
     type: 'worker';

package/build/types/durable.d.ts

@@ -1,5 +1,6 @@
 import { LogLevel } from './logger';
 import { RedisClass, RedisOptions } from './redis';
+import { StringStringType } from './serializer';
 type WorkflowConfig = {
     backoffCoefficient?: number;
     maximumAttempts?: number;
@@ -11,6 +12,14 @@ type WorkflowContext = {
      * the reentrant semaphore, incremented in real-time as idempotent statements are re-traversed upon reentry. Indicates the current semaphore count.
      */
     counter: number;
+    /**
+     * number as string for the replay cursor
+     */
+    cursor: string;
+    /**
+     * the replay hash of name/value pairs representing prior executions
+     */
+    replay: StringStringType;
     /**
      * the HotMesh App namespace. `durable` is the default.
      */

package/build/types/exporter.d.ts

@@ -21,6 +21,8 @@ export interface JobTimeline {
     dimension: string;
     duplex: 'entry' | 'exit';
     timestamp: string;
+    created?: string;
+    updated?: string;
     actions?: ActivityAction[];
 }
 export interface DependencyExport {

package/build/types/job.d.ts

@@ -15,6 +15,7 @@ type JobMetadata = {
     pg?: string;
     pd?: string;
     pa?: string;
+    px?: boolean;
     ngn?: string;
     app: string;
     vrs: string;

package/build/types/quorum.d.ts

@@ -45,42 +45,56 @@ export interface QuorumProfile {
     reclaimDelay?: number;
     reclaimCount?: number;
     system?: SystemHealth;
+    signature?: string;
 }
-export interface PingMessage {
+interface QuorumMessageBase {
+    guid?: string;
+    topic?: string;
+    type?: string;
+}
+export interface PingMessage extends QuorumMessageBase {
     type: 'ping';
     originator: string;
     details?: boolean;
 }
-export interface WorkMessage {
+export interface WorkMessage extends QuorumMessageBase {
     type: 'work';
     originator: string;
 }
-export interface CronMessage {
+export interface CronMessage extends QuorumMessageBase {
     type: 'cron';
     originator: string;
 }
-export interface PongMessage {
+export interface PongMessage extends QuorumMessageBase {
     type: 'pong';
     guid: string;
     originator: string;
     profile?: QuorumProfile;
 }
-export interface ActivateMessage {
+export interface ActivateMessage extends QuorumMessageBase {
     type: 'activate';
     cache_mode: 'nocache' | 'cache';
     until_version: string;
 }
-export interface JobMessage {
+export interface JobMessage extends QuorumMessageBase {
     type: 'job';
     topic: string;
     job: JobOutput;
 }
-export interface ThrottleMessage {
+export interface ThrottleMessage extends QuorumMessageBase {
     type: 'throttle';
     guid?: string;
     topic?: string;
     throttle: number;
 }
+export interface RollCallMessage extends QuorumMessageBase {
+    type: 'rollcall';
+    guid?: string;
+    topic?: string | null;
+    interval: number;
+    max?: number;
+    signature?: boolean;
+}
 export interface JobMessageCallback {
     (topic: string, message: JobOutput): void;
 }
@@ -96,5 +110,5 @@ export interface QuorumMessageCallback {
  * These messages serve to coordinate the cache invalidation and switch-over
  * to the new version without any downtime and a coordinating parent server.
  */
-export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | RollCallMessage | CronMessage;
 export {};

package/build/types/stream.d.ts

@@ -37,6 +37,7 @@ export interface StreamData {
         trc?: string;
         spn?: string;
         try?: number;
+        await?: boolean;
     };
     type?: StreamDataType;
     data: Record<string, unknown>;

package/modules/enums.ts

@@ -23,6 +23,7 @@ export const HMSH_CODE_DURABLE_RETRYABLE = 599;
 export const HMSH_STATUS_UNKNOWN = 'unknown';
 
 // QUORUM
+export const HMSH_QUORUM_ROLLCALL_CYCLES = 12; //max iterations
 export const HMSH_QUORUM_DELAY_MS = 250;
 export const HMSH_ACTIVATION_MAX_RETRY = 3;
 

package/modules/key.ts

@@ -28,7 +28,12 @@ import { KeyStoreParams, KeyType } from '../types/hotmesh';
  * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
  */
 
-const HMNS = "hmsh"; //default
+const HMNS = "hmsh";
+
+const KEYSEP = ':';  //default delimiter for keys 
+const VALSEP = '::'; //default delimiter for vals
+const WEBSEP = '::'; //default delimiter for webhook vals
+const TYPSEP = '::'; //delimiter for ZSET task typing (how should a list be used?)
 
 class KeyService {
 
@@ -93,4 +98,4 @@ class KeyService {
   }
 }
 
-export { KeyService, KeyType, KeyStoreParams, HMNS };
+export { KeyService, KeyType, KeyStoreParams, HMNS, KEYSEP, TYPSEP, WEBSEP, VALSEP };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.48",
+  "version": "0.0.50",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -28,6 +28,7 @@
     "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:emit": "NODE_ENV=test jest ./tests/functional/emit/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:await": "NODE_ENV=test jest ./tests/functional/awaiter/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:hook": "NODE_ENV=test jest ./tests/functional/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:signal": "NODE_ENV=test jest ./tests/functional/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:interrupt": "NODE_ENV=test jest ./tests/functional/interrupt/index.test.ts --detectOpenHandles --forceExit --verbose",

package/services/activities/await.ts

@@ -95,6 +95,12 @@ class Await extends Activity {
       type: StreamDataType.AWAIT,
       data: this.context.data
     };
+    if (this.config.await !== true) {
+      const doAwait = Pipe.resolve(this.config.await, this.context);
+      if (doAwait === false) {
+        streamData.metadata.await = false;
+      }
+    }
     if (this.config.retry) {
       streamData.policies = {
         retry: this.config.retry

package/services/activities/hook.ts

@@ -148,6 +148,7 @@ class Hook extends Activity {
         `${this.metadata.aid}${this.metadata.dad || ''}`,
         'sleep',
         duration,
+        this.metadata.dad || '',
       );
       return this.context.metadata.jid;
     }

package/services/activities/trigger.ts

@@ -15,6 +15,7 @@ import {
 import { JobState } from '../../types/job';
 import { RedisMulti } from '../../types/redis';
 import { StringScalarType } from '../../types/serializer';
+import { WorkListTaskType } from '../../types/task';
 
 class Trigger extends Activity {
   config: TriggerActivity;
@@ -50,6 +51,10 @@ class Trigger extends Activity {
       await this.registerJobDependency(multi);
       await multi.exec();
 
+      //if the parent (spawner) chose not to await,
+      // emit the job_id as the data payload { job_id }
+      this.execAdjacentParent();
+
       telemetry.mapActivityAttributes();
       const jobStatus = Number(this.context.metadata.js);
       telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
@@ -79,6 +84,12 @@ class Trigger extends Activity {
     this.context.metadata.js = amount;
   }
 
+  async execAdjacentParent() {
+    if (this.context.metadata.px) {
+      await this.engine.execAdjacentParent(this.context, {metadata: this.context.metadata, data: { job_id: this.context.metadata.jid }});
+    }
+  }
+
   createInputContext(): Partial<JobState> {
     const input = { 
       [this.metadata.aid]: {
@@ -115,6 +126,7 @@ class Trigger extends Activity {
         pg: this.context.metadata.pg,
         pd: this.context.metadata.pd,
         pa: this.context.metadata.pa,
+        px: this.context.metadata.px,
         app: id,
         vrs: version,
         tpc: this.config.subscribes,
@@ -193,12 +205,23 @@ class Trigger extends Activity {
     }
     if (resolvedDepKey) {
       const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
+      let type: WorkListTaskType;
+      if (isParentOrigin) {
+        if (this.context.metadata.px) {
+          type = 'child'
+        } else {
+          type = 'expire-child'
+        }
+      } else {
+        type = 'expire';
+      }
       await this.store.registerJobDependency(
-        isParentOrigin ? 'expire-child' : 'expire',
+        type,
         resolvedDepKey,
         this.context.metadata.tpc,
         this.context.metadata.jid,
         this.context.metadata.gid,
+        this.context.metadata.pd,
         multi,
       );
     }
@@ -209,6 +232,7 @@ class Trigger extends Activity {
         this.context.metadata.tpc,
         this.context.metadata.jid,
         this.context.metadata.gid,
+        this.context.metadata.pd,
         multi,
       );
     }

package/services/durable/exporter.ts

@@ -13,6 +13,7 @@ import {
   JobTimeline } from '../../types/exporter';
 import { SerializerService } from '../serializer';
 import { restoreHierarchy } from '../../modules/utils';
+import { VALSEP } from '../../modules/key';
 
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
@@ -116,19 +117,31 @@ class ExporterService {
       const activityName = item[1].split('/')[0];
       const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
       const timestamp = item[2];
-      const event: JobTimeline = {
+      let event: JobTimeline = {
         activity: activityName,
         duplex: duplex as 'entry' | 'exit',
         dimension: dimensions,
         timestamp,
+        created: timestamp,
+        updated: timestamp,
       };
-      timeline.push(event);
+      const prior = timeline[timeline.length - 1];
+      if (prior && prior.activity === event.activity && prior.duplex !== event.duplex && prior.dimension === event.dimension) {
+        if (event.duplex === 'exit') {
+          prior.updated = event.timestamp;
+        } else {
+          prior.created = event.timestamp;
+        }
+        event = prior;
+      } else {
+        timeline.push(event);
+      }
 
       if (this.isMainEntry(item[1])) {
         event.actions = [] as ActivityAction[];
         this.interleaveActions(actions.main, event.actions);
       } else if (this.isHookEntry(item[1])) {
-        const hookDimension =  `/${parts[1]}/${parts[2]}`;
+        const hookDimension = `/${parts[1]}/${parts[2]}`;
         const hookActions = actions.hooks[hookDimension];
         event.actions = [] as ActivityAction[];
         this.interleaveActions(hookActions, event.actions);
@@ -191,17 +204,15 @@ class ExporterService {
    * @returns - the organized dependency data
    */
   inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
-    //console.log('dependency data>', data);
     const hookReg = /([0-9,]+)-(\d+)$/;
     const flowReg = /-(\d+)$/;
     return data.map((dependency, index: number): DependencyExport => {
-      const [action, topic, gid, ...jid] = dependency.split('::');
-      const jobId = jid.join('::');
+      const [action, topic, gid, _pd, ...jid] = dependency.split(VALSEP);
+      const jobId = jid.join(VALSEP);
       const match = jobId.match(hookReg);
       let prefix: string;
       let type: 'hook' | 'flow' | 'other';
       let dimensionKey: string = '';
-
       if (match) {
         //hook-originating dependency
         const [_, dimension, counter] = match;

package/services/durable/meshos.ts

@@ -288,13 +288,17 @@ export class MeshOSService {
       args.push('LIMIT', '0', '0');
     } else {
       //limit which hash fields to return
-      if (options.return?.length) {
-        args.push('RETURN');
-        args.push(options.return.length.toString());
-        options.return.forEach(returnField => {
+      args.push('RETURN');
+      args.push(((options.return?.length ?? 0) + 1).toString());
+      args.push('$');
+      options.return?.forEach(returnField => {
+        if (returnField.startsWith('"')) {
+          //allow literal values to be requested
+          args.push(returnField.slice(1, -1));
+        } else {
           args.push(`_${returnField}`);
-        });
-      }
+        }
+      });
       //paginate
       if (options.limit) {
         args.push('LIMIT', options.limit.start.toString(), options.limit.size.toString());

package/services/durable/search.ts

@@ -3,6 +3,7 @@ import { RedisClient, RedisMulti } from '../../types/redis';
 import { StoreService } from '../store';
 import { KeyService, KeyType } from '../../modules/key';
 import { WorkflowSearchOptions } from '../../types/durable';
+import { asyncLocalStorage } from '../../modules/storage';
 
 export class Search {
   jobId: string;
@@ -66,9 +67,14 @@ export class Search {
    * @returns {Promise<string[]>} - the list of search indexes
    */
   static async listSearchIndexes(hotMeshClient: HotMesh): Promise<string[]> {
-    const store = hotMeshClient.engine.store;
-    const searchIndexes = await store.exec('FT._LIST');
-    return searchIndexes as string[];
+    try {
+      const store = hotMeshClient.engine.store;
+      const searchIndexes = await store.exec('FT._LIST');
+      return searchIndexes as string[];
+    } catch (err) {
+      hotMeshClient.engine.logger.info('durable-client-search-list-err', { err });
+      return [];
+    }
   }
 
   /**
@@ -80,18 +86,28 @@ export class Search {
     return `${this.searchSessionId}-${this.searchSessionIndex++}-`;
   }
 
-  async set(...args: string[]): Promise<void> {
+  /**
+   * Sets the fields listed in args. Returns the
+   * count of new fields that were set (does not
+   * count fields that were updated)
+   */
+  async set(...args: string[]): Promise<number> {
     const ssGuid = this.getSearchSessionGuid();
-    const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
-    if (ssGuidValue === 1) {
-      const safeArgs: string[] = [];
-      for (let i = 0; i < args.length; i += 2) {
-        const key = this.safeKey(args[i]);
-        const value = args[i+1].toString();
-        safeArgs.push(key, value);
-      }
-      await this.store.exec('HSET', this.jobId, ...safeArgs);
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Number(replay[ssGuid]);
     }
+    const safeArgs: string[] = [];
+    for (let i = 0; i < args.length; i += 2) {
+      const key = this.safeKey(args[i]);
+      const value = args[i+1].toString();
+      safeArgs.push(key, value);
+    }
+    const fieldCount = await this.store.exec('HSET', this.jobId, ...safeArgs);
+    //no need to wait; set this interim value in the replay
+    this.store.exec('HSET', this.jobId, ssGuid, fieldCount.toString());
+    return Number(fieldCount);
   }
 
   async get(key: string): Promise<string> {
@@ -116,34 +132,65 @@ export class Search {
     }
   }
 
+  /**
+   * Deletes the fields listed in args. Returns the
+   * count of fields that were deleted.
+   */
   async del(...args: string[]): Promise<number | void> {
     const ssGuid = this.getSearchSessionGuid();
-    const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
-    if (ssGuidValue === 1) {
-      const safeArgs: string[] = [];
-      for (let i = 0; i < args.length; i++) {
-        safeArgs.push(this.safeKey(args[i]));
-      }
-      const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
-      return isNaN(response as unknown as number) ? undefined : Number(response);
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Number(replay[ssGuid]);
+    }
+    const safeArgs: string[] = [];
+    for (let i = 0; i < args.length; i++) {
+      safeArgs.push(this.safeKey(args[i]));
     }
+    const response = await this.store.exec('HDEL', this.jobId, ...safeArgs);
+    const formattedResponse = isNaN(response as unknown as number) ? 0 : Number(response);
+    //no need to wait; set this interim value in the replay
+    this.store.exec('HSET', this.jobId, ssGuid, formattedResponse.toString());
+    return formattedResponse;
   }
 
+  /**
+   * Increments the value of a field by the given amount. Returns the
+   * new value of the field after the increment. Can be
+   * used to decrement the value of a field by specifying a negative.
+   */
   async incr(key: string, val: number): Promise<number> {
     const ssGuid = this.getSearchSessionGuid();
-    const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
-    if (ssGuidValue === 1) {
-      return Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string);
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Number(replay[ssGuid]);
     }
+    const num = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), val.toString()) as string;
+    //no need to wait; set this interim value in the replay
+    this.store.exec('HSET', this.jobId, ssGuid, num.toString());
+    return Number(num);
   }
 
+  /**
+   * Multiplies the value of a field by the given amount. Returns the
+   * new value of the field after the multiplication. NOTE:
+   * this is exponential multiplication.
+   */
   async mult(key: string, val: number): Promise<number> {
     const ssGuid = this.getSearchSessionGuid();
+    const store = asyncLocalStorage.getStore();
+    const replay = store?.get('replay') ?? {};
+    if (ssGuid in replay) {
+      return Math.exp(Number(replay[ssGuid]));
+    }
     const ssGuidValue = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, ssGuid, '1') as string);
     if (ssGuidValue === 1) {
       const log = Math.log(val);
-      const logTotal = Number(await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string);
-      return Math.exp(logTotal);
+      const logTotal = await this.store.exec('HINCRBYFLOAT', this.jobId, this.safeKey(key), log.toString()) as string;
+      //no need to wait; set this interim value in the replay
+      this.store.exec('HSET', this.jobId, ssGuid, logTotal.toString());
+      return Math.exp(Number(logTotal));
     }
   }
 }

package/services/durable/worker.ts

@@ -211,16 +211,29 @@ export class WorkerService {
           // garbage collect (expire) this job when originJobId is expired
           context.set('originJobId', workflowInput.originJobId);
         }
+        let replayQuery = '';
         if (workflowInput.workflowDimension) {
           //every hook function runs in an isolated dimension controlled
           //by the index assigned when the signal was received; even if the
           //hook function re-runs, its scope will always remain constant
           context.set('workflowDimension', workflowInput.workflowDimension);
+          replayQuery = `-*${workflowInput.workflowDimension}-*`;
+        } else {
+          //last letter of words like 'hook', 'sleep', 'wait', 'signal', 'search', 'start'
+          replayQuery = '-*[ehklpt]-*';
         }
         context.set('workflowTopic', workflowTopic);
         context.set('workflowName', workflowTopic.split('-').pop());
         context.set('workflowTrace', data.metadata.trc);
         context.set('workflowSpan', data.metadata.spn);
+        const store = this.workflowRunner.engine.store;
+        const [cursor, replay] = await store.findJobFields(
+          workflowInput.workflowId,
+          replayQuery,
+          50_000,
+          5_000,);
+        context.set('replay', replay);
+        context.set('cursor', cursor); // if != 0, more remain
         const workflowResponse = await asyncLocalStorage.run(context, async () => {
           return await workflowFunction.apply(this, workflowInput.arguments);
         });
@@ -232,7 +245,6 @@ export class WorkerService {
           data: { response: workflowResponse, done: true }
         };
       } catch (err) {
-
         //not an error...just a trigger to sleep
         if (err instanceof DurableSleepForError) {
           return {