@hotmeshio/hotmesh 0.0.41 → 0.0.43

package/services/durable/factory.ts

@@ -7,7 +7,6 @@
  * ERROR CODES:
  *      594: waitforsignal
  *      592: sleepFor
- *      595: sleep (deprecated)
  *      596, 597, 598: fatal
  *      599: retry
  */
@@ -122,19 +121,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               maps:
                 index: '{$self.output.data.index}'
                 signals: '{$self.output.data.signals}'
-            595:
-              schema:
-                type: object
-                properties:
-                  duration:
-                    type: number
-                    description: sleep duration in seconds
-                  index:
-                    type: number
-                    description: the current index
-              maps:
-                duration: '{$self.output.data.duration}'
-                index: '{$self.output.data.index}'
             592:
               schema:
                 type: object
@@ -229,19 +215,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               maps:
                 index: '{$self.output.data.index}'
                 signals: '{$self.output.data.signals}'
-            595:
-              schema:
-                type: object
-                properties:
-                  duration:
-                    type: number
-                    description: sleep duration in seconds
-                  index:
-                    type: number
-                    description: the current index
-              maps:
-                duration: '{$self.output.data.duration}'
-                index: '{$self.output.data.index}'
             592:
               schema:
                 type: object
@@ -320,57 +293,6 @@ const getWorkflowYAML = (app: string, version: string) => {
             maps:
               duration: '{siga1.output.data.duration}'
 
-        siga595:
-          title: Signal In - Sleep before trying again
-          type: await
-          topic: ${app}.sleep.execute
-          input:
-            schema:
-              type: object
-              properties:
-                duration:
-                  type: number
-                index:
-                  type: number
-                workflowId:
-                  type: string
-                parentWorkflowId:
-                  type: string
-                originJobId:
-                  type: string
-            maps:
-              duration: '{sigw1.output.data.duration}'
-              index: '{sigw1.output.data.index}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-s']
-                  - ['{@string.concat}']
-              originJobId:
-                '@pipe':
-                  - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
-                  - ['{@conditional.ternary}']
-
-              workflowId:
-                '@pipe':
-                  - ['-', '{$job.metadata.jid}', '-$sleep', '{sig.output.metadata.dad}', '-', '{sigw1.output.data.index}']
-                  - ['{@string.concat}']
-          output:
-            schema:
-              type: object
-              properties:
-                done:
-                  type: boolean
-            maps:
-              done: '{sigw1.output.data.done}'
-
-        sigc595:
-          title: Signal In - Goto Activity siga1
-          type: cycle
-          ancestor: siga1
-          input:
-            maps:
-              duration: '{siga1.output.data.duration}'
-
         siga592:
           title: Signal In - Sleep For a duration and then cycle/goto
           type: hook
@@ -464,56 +386,6 @@ const getWorkflowYAML = (app: string, version: string) => {
             maps:
               duration: '{a1.output.data.duration}'
 
-        a595:
-          title: Sleep before trying again
-          type: await
-          topic: ${app}.sleep.execute
-          input:
-            schema:
-              type: object
-              properties:
-                duration:
-                  type: number
-                index:
-                  type: number
-                workflowId:
-                  type: string
-                parentWorkflowId:
-                  type: string
-                originJobId:
-                  type: string
-            maps:
-              duration: '{w1.output.data.duration}'
-              index: '{w1.output.data.index}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-s']
-                  - ['{@string.concat}']
-              originJobId:
-                '@pipe':
-                  - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
-                  - ['{@conditional.ternary}']
-              workflowId:
-                '@pipe':
-                  - ['-', '{$job.metadata.jid}', '-$sleep-', '{w1.output.data.index}']
-                  - ['{@string.concat}']
-          output:
-            schema:
-              type: object
-              properties:
-                done:
-                  type: boolean
-            maps:
-              done: '{w1.output.data.done}'
-
-        c595:
-          title: Goto Activity a1
-          type: cycle
-          ancestor: a1
-          input:
-            maps:
-              duration: '{a1.output.data.duration}'
-
         a592:
           title: Sleep For a duration and then cycle/goto
           type: hook
@@ -571,9 +443,6 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: siga594
             conditions:
               code: 594
-          - to: siga595
-            conditions:
-              code: 595
           - to: siga592
             conditions:
               code: 592
@@ -582,8 +451,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               code: 599
         siga594:
           - to: sigc594
-        siga595:
-          - to: sigc595
         siga592:
           - to: sigc592
         siga599:
@@ -594,9 +461,6 @@ const getWorkflowYAML = (app: string, version: string) => {
           - to: a594
             conditions:
               code: 594
-          - to: a595
-            conditions:
-              code: 595
           - to: a592
             conditions:
               code: 592
@@ -608,8 +472,6 @@ const getWorkflowYAML = (app: string, version: string) => {
               code: [200, 598, 597, 596]
         a594:
           - to: c594
-        a595:
-          - to: c595
         a592:
           - to: c592
         a599:

package/services/durable/meshos.ts

@@ -295,6 +295,9 @@ export class MeshOSService {
   }
 
   static generateSearchQuery(query: FindWhereQuery[]) {
+    if (!Array.isArray(query) || query.length === 0) {
+      return '*';
+    }
     const my = new this();
     let queryString = query.map(q => {
       const { field, is, value, type } = q;

package/services/durable/worker.ts

@@ -3,7 +3,6 @@ import {
   DurableIncompleteSignalError,
   DurableMaxedError,
   DurableRetryError,
-  DurableSleepError,
   DurableSleepForError,
   DurableTimeoutError, 
   DurableWaitForSignalError} from '../../modules/errors';
@@ -249,21 +248,6 @@ export class WorkerService {
             }
           } as StreamDataResponse;
 
-        //deprecated format; not an error...just a trigger to sleep
-        } else if (err instanceof DurableSleepError) {
-          return {
-            status: StreamStatus.SUCCESS,
-            code: err.code,
-            metadata: { ...data.metadata },
-            data: {
-              code: err.code,
-              message: JSON.stringify({ duration: err.duration, index: err.index, dimension: err.dimension }),
-              duration: err.duration,
-              index: err.index,
-              dimension: err.dimension
-            }
-          } as StreamDataResponse;
-
         //not an error...just a trigger to wait for a signal
         } else if (err instanceof DurableWaitForSignalError) {
           return {

package/services/durable/workflow.ts

@@ -2,7 +2,6 @@ import ms from 'ms';
 
 import {
   DurableIncompleteSignalError,
-  DurableSleepError,
   DurableSleepForError,
   DurableWaitForSignalError } from '../../modules/errors';
 import { KeyService, KeyType } from '../../modules/key';
@@ -376,37 +375,6 @@ export class WorkflowService {
     return seconds;
   }
 
-  /**
-   * Sleeps the workflow for a duration. As the function is reentrant, 
-   * upon reentry, the function will traverse prior execution paths up
-   * until the sleep command and then resume execution from that point.
-   * @param {string} duration - for example: '1 minute', '2 hours', '3 days'
-   * @returns {Promise<number>}
-   * @deprecated - use `sleepFor` instead
-   */
-  static async sleep(duration: string): Promise<number> {
-    const seconds = ms(duration) / 1000;
-
-    const store = asyncLocalStorage.getStore();
-    const COUNTER = store.get('counter');
-    const execIndex = COUNTER.counter = COUNTER.counter + 1;
-    const workflowId = store.get('workflowId');
-    const workflowTopic = store.get('workflowTopic');
-    const workflowDimension = store.get('workflowDimension') ?? '';
-    const namespace = store.get('namespace');
-    const sleepJobId = `-${workflowId}-$sleep${workflowDimension}-${execIndex}`;
-
-    try {
-      const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
-      await hotMeshClient.getState(`${hotMeshClient.appId}.sleep.execute`, sleepJobId);
-      //if no error is thrown, we've already slept, return the delay
-      return seconds;
-    } catch (e) {
-      // spawn a new sleep job if error code 595 is thrown by the worker)
-      throw new DurableSleepError(workflowId, seconds, execIndex, workflowDimension);
-    }
-  }
-
   /**
    * Waits for a signal to awaken
    * @param {string[]} signals - the signals to wait for

package/services/engine/index.ts

@@ -332,6 +332,7 @@ class EngineService {
     });
     const context: PartialJobState = {
       metadata: {
+        guid: streamData.metadata.guid,
         jid: streamData.metadata.jid,
         gid: streamData.metadata.gid,
         dad: streamData.metadata.dad,
@@ -490,17 +491,15 @@ class EngineService {
     };
     return await this.router.publishMessage(null, streamData) as string;
   }
-  async hookTime(jobId: string, gId: string, activityId: string, type?: WorkListTaskType): Promise<string | void> {
-    if (type === 'interrupt') {
+  async hookTime(jobId: string, gId: string, topicOrActivity: string, type?: WorkListTaskType): Promise<string | void> {
+    if (type === 'interrupt' || type === 'expire') {
       return await this.interrupt(
-        activityId, //note: 'activityId' is the actually job topic
+        topicOrActivity,
         jobId,
         { suppress: true, expire: 1 },
       );
-    } else if (type === 'expire') {
-      return await this.store.expireJob(jobId, 1);
     }
-    const [aid, ...dimensions] = activityId.split(',');
+    const [aid, ...dimensions] = topicOrActivity.split(',');
     const dad = `,${dimensions.join(',')}`;
     const streamData: StreamData = {
       type: StreamDataType.TIMEHOOK,

package/services/quorum/index.ts

@@ -1,4 +1,4 @@
-import { KeyType } from '../../modules/key';
+import { HMSH_ACTIVATION_MAX_RETRY, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
 import { identifyRedisType, sleepFor } from '../../modules/utils';
 import { CompilerService } from '../compiler';
 import { EngineService } from '../engine';
@@ -10,6 +10,7 @@ import { SubService } from '../sub';
 import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
 import { RedisSubService as RedisSub } from '../sub/clients/redis';
 import { CacheMode } from '../../types/cache';
+import { HotMeshConfig, KeyType } from '../../types/hotmesh';
 import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
 import {
   QuorumMessage,
@@ -18,13 +19,9 @@ import {
   SubscriptionCallback,
   ThrottleMessage
 } from '../../types/quorum';
-import { HotMeshApps, HotMeshConfig } from '../../types/hotmesh';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { RedisClientType } from '../../types/redisclient';
 
-//wait time to see if quorum is reached
-const QUORUM_DELAY = 250;
-
 class QuorumService {
   namespace: string;
   appId: string;
@@ -59,8 +56,18 @@ class QuorumService {
       //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
       await instance.initStoreChannel(config.engine.store);
       await instance.initSubChannel(config.engine.sub);
-      await instance.subscribe.subscribe(KeyType.QUORUM, instance.subscriptionHandler(), appId); //general quorum subscription
-      await instance.subscribe.subscribe(KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid); //app-specific quorum subscription (used for pubsub one-time request/response)
+      //general quorum subscription
+      await instance.subscribe.subscribe(
+        KeyType.QUORUM,
+        instance.subscriptionHandler(),
+        appId
+      );
+      //app-specific quorum subscription (used for pubsub one-time request/response)
+      await instance.subscribe.subscribe(
+        KeyType.QUORUM,
+        instance.subscriptionHandler(),
+        appId, instance.guid
+      );
       
       instance.engine.processWebHooks();
       instance.engine.processTimeHooks();
@@ -152,7 +159,7 @@ class QuorumService {
     );
   }
 
-  async requestQuorum(delay = QUORUM_DELAY, details = false): Promise<number> {
+  async requestQuorum(delay = HMSH_QUORUM_DELAY_MS, details = false): Promise<number> {
     const quorum = this.quorum;
     this.quorum = 0;
     this.profiles.length = 0;
@@ -188,7 +195,7 @@ class QuorumService {
 
 
   // ************* COMPILER METHODS *************
-  async rollCall(delay = QUORUM_DELAY): Promise<QuorumProfile[]> {
+  async rollCall(delay = HMSH_QUORUM_DELAY_MS): Promise<QuorumProfile[]> {
     await this.requestQuorum(delay, true);
     const targetStreams = [];
     const multi = this.store.getMulti();
@@ -209,10 +216,20 @@ class QuorumService {
     });
     return this.profiles;
   }
-  async activate(version: string, delay = QUORUM_DELAY): Promise<boolean> {
+  /**
+   * request a quorum; if successful activate the app version
+   */
+  async activate(version: string, delay = HMSH_QUORUM_DELAY_MS, count = 0): Promise<boolean> {
     version = version.toString();
+    const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
+    if (!canActivate) {
+      //another engine is already activating the app version
+      this.logger.debug('quorum-activation-awaiting', { version });
+      await sleepFor(delay * 6);
+      const app = await this.store.getApp(this.appId, true);
+      return app?.active == true && app?.version === version;
+    }
     const config = await this.engine.getVID();
-    //request a quorum to activate the version
     await this.requestQuorum(delay);
     const q1 = await this.requestQuorum(delay);
     const q2 = await this.requestQuorum(delay);
@@ -225,6 +242,7 @@ class QuorumService {
         this.appId
       );
       await new Promise(resolve => setTimeout(resolve, delay));
+      await this.store.releaseScoutRole('activate');
       //confirm we received the activation message
       if (this.engine.untilVersion === version) {
         this.logger.info('quorum-activation-succeeded', { version });
@@ -236,7 +254,12 @@ class QuorumService {
         throw new Error(`UntilVersion Not Received. Version ${version} not activated`);
       }
     } else {
-      this.logger.info('quorum-rollcall-error', { q1, q2, q3 });
+      this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
+      this.store.releaseScoutRole('activate');
+      if (count < HMSH_ACTIVATION_MAX_RETRY) {
+        //increase the delay (give the quorum time to respond) and try again
+        return await this.activate(version, delay * 2, count + 1);
+      }
       throw new Error(`Quorum not reached. Version ${version} not activated.`);
     }
   }

package/services/store/clients/redis.ts

@@ -18,6 +18,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
   constructor(redisClient: RedisClientType) {
     super(redisClient);
     this.commands = {
+      set: 'SET',
       setnx: 'SETNX',
       del: 'DEL',
       expire: 'EXPIRE',

package/services/store/index.ts

@@ -45,6 +45,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
   appId: string
   logger: ILogger;
   commands: Record<string, string> = {
+    set: 'set',
     setnx: 'setnx',
     del: 'del',
     expire: 'expire',
@@ -178,14 +179,16 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * check for and process work items in the
    * time and signal task queues.
    */
-  async reserveScoutRole(scoutType: 'time' | 'signal', delay = HMSH_SCOUT_INTERVAL_SECONDS): Promise<boolean> {
+  async reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay = HMSH_SCOUT_INTERVAL_SECONDS): Promise<boolean> {
     const key = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
-    const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${formatISODate(new Date())}`);
-    if (this.isSuccessful(success)) {
-      await this.redisClient[this.commands.expire](key, delay - 1);
-      return true;
-    }
-    return false;
+    const success = await this.exec('SET', key, `${scoutType}:${formatISODate(new Date())}`, 'NX', 'EX', `${delay - 1}`);
+    return this.isSuccessful(success);
+  }
+
+  async releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean> {
+    const key = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
+    const success = await this.exec('DEL', key);
+    return this.isSuccessful(success);
   }
 
   async getSettings(bCreate = false): Promise<HotMeshSettings> {
@@ -586,6 +589,10 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
+  /**
+   * collate is a generic method for incrementing a value in a hash
+   * in order to track their progress during processing.
+   */
   async collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi? : U): Promise<number> {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     const collationKey = `${activityId}/output/metadata/as`; //activity state
@@ -600,6 +607,17 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
   }
 
+  /**
+   * synthentic collation affects those activities in the graph
+   * that represent the synthetic DAG that was materialized during compilation;
+   * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+   * purposeful re-entry.
+   */
+  async collateSynthetic(jobId: string, guid: string, amount: number, multi? : U): Promise<number> {
+    const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+    return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
+  }
+
   async setStateNX(jobId: string, appId: string): Promise<boolean> {
     const hashKey = this.mintKey(KeyType.JOB_STATE, { appId, jobId });
     const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', '1');

package/types/hotmesh.ts

@@ -44,7 +44,7 @@ type KeyStoreParams = {
   facet?: string;       //data path starting at root with values separated by colons (e.g. "object/type:bar")
   topic?: string;       //topic name (e.g., "foo" or "" for top-level)
   timeValue?: number;   //time value (rounded to minute) (for delete range)
-  scoutType?: 'signal' | 'time'; //a single member of the quorum serves as the 'scout' for the group, triaging tasks for the collective
+  scoutType?: 'signal' | 'time' | 'activate'; //a single member of the quorum serves as the 'scout' for the group, triaging tasks for the collective
 };
 
 type HotMesh = typeof HotMeshService;

package/types/job.ts

@@ -8,6 +8,7 @@ type ActivityData = {
 
 type JobMetadata = {
   key?: string; //job_key
+  guid?: string; //system assigned guid that corresponds to the transition message guid that spawned reentry
   gid: string;  //system assigned guid; ensured created/deleted/created jobs are unique
   jid: string;  //job_id (jid+dad+aid) is composite key for activity
   dad: string;  //dimensional address for the activity (,0,0,1)