@hotmeshio/hotmesh 0.0.42 → 0.0.43

package/build/modules/enums.d.ts

@@ -15,6 +15,8 @@ export declare const HMSH_CODE_DURABLE_MAXED = 597;
 export declare const HMSH_CODE_DURABLE_FATAL = 598;
 export declare const HMSH_CODE_DURABLE_RETRYABLE = 599;
 export declare const HMSH_STATUS_UNKNOWN = "unknown";
+export declare const HMSH_QUORUM_DELAY_MS = 250;
+export declare const HMSH_ACTIVATION_MAX_RETRY = 3;
 export declare const HMSH_OTT_WAIT_TIME: number;
 export declare const HMSH_EXPIRE_JOB_SECONDS: number;
 export declare const HMSH_MAX_RETRIES: number;

package/build/modules/enums.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
+exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
 // HOTMESH SYSTEM
 exports.HMSH_LOGLEVEL = process.env.HMSH_LOGLEVEL || 'info';
 // STATUS CODES AND MESSAGES
@@ -19,6 +19,9 @@ exports.HMSH_CODE_DURABLE_MAXED = 597;
 exports.HMSH_CODE_DURABLE_FATAL = 598;
 exports.HMSH_CODE_DURABLE_RETRYABLE = 599;
 exports.HMSH_STATUS_UNKNOWN = 'unknown';
+// QUORUM
+exports.HMSH_QUORUM_DELAY_MS = 250;
+exports.HMSH_ACTIVATION_MAX_RETRY = 3;
 // ENGINE
 exports.HMSH_OTT_WAIT_TIME = parseInt(process.env.HMSH_OTT_WAIT_TIME, 10) || 1000;
 exports.HMSH_EXPIRE_JOB_SECONDS = parseInt(process.env.HMSH_EXPIRE_JOB_SECONDS, 10) || 1;

package/build/modules/utils.js

@@ -7,7 +7,7 @@ async function sleepFor(ms) {
 }
 exports.sleepFor = sleepFor;
 function guid() {
-    return (0, nanoid_1.nanoid)();
+    return (0, nanoid_1.nanoid)().replace(/[_-]/g, '0');
 }
 exports.guid = guid;
 function deterministicRandom(seed) {

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.42",
+    "version": "0.0.43",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/durable/client.d.ts

@@ -3,8 +3,8 @@ import { HotMeshService as HotMesh } from '../hotmesh';
 import { ClientConfig, Connection, HookOptions, WorkflowOptions, WorkflowSearchOptions } from '../../types/durable';
 export declare class ClientService {
     connection: Connection;
-    topics: string[];
     options: WorkflowOptions;
+    static topics: string[];
     static instances: Map<string, HotMesh | Promise<HotMesh>>;
     constructor(config: ClientConfig);
     getHotMeshClient: (workflowTopic: string, namespace?: string) => Promise<HotMesh>;
@@ -37,6 +37,7 @@ export declare class ClientService {
         getHandle: (taskQueue: string, workflowName: string, workflowId: string, namespace?: string) => Promise<WorkflowHandleService>;
         search: (taskQueue: string, workflowName: string, namespace: null | string, index: string, ...query: string[]) => Promise<string[]>;
     };
+    verifyWorkflowActive(hotMesh: HotMesh, appId?: string, count?: number): Promise<boolean>;
     activateWorkflow(hotMesh: HotMesh, appId?: string, version?: string): Promise<void>;
     static shutdown(): Promise<void>;
 }

package/build/services/durable/client.js

@@ -8,16 +8,17 @@ const key_1 = require("../../modules/key");
 const search_1 = require("./search");
 const types_1 = require("../../types");
 const enums_1 = require("../../modules/enums");
+const utils_1 = require("../../modules/utils");
 class ClientService {
     constructor(config) {
-        this.topics = [];
         this.getHotMeshClient = async (workflowTopic, namespace) => {
             //use the cached instance
             const instanceId = 'SINGLETON';
             if (ClientService.instances.has(instanceId)) {
                 const hotMeshClient = await ClientService.instances.get(instanceId);
-                if (!this.topics.includes(workflowTopic)) {
-                    this.topics.push(workflowTopic);
+                await this.verifyWorkflowActive(hotMeshClient, namespace ?? factory_1.APP_ID);
+                if (!ClientService.topics.includes(workflowTopic)) {
+                    ClientService.topics.push(workflowTopic);
                     await this.createStream(hotMeshClient, workflowTopic, namespace);
                 }
                 return hotMeshClient;
@@ -174,6 +175,18 @@ class ClientService {
         };
         this.connection = config.connection;
     }
+    async verifyWorkflowActive(hotMesh, appId = factory_1.APP_ID, count = 0) {
+        const app = await hotMesh.engine.store.getApp(appId);
+        const appVersion = app?.version;
+        if (isNaN(appVersion)) {
+            if (count > 10) {
+                throw new Error('Workflow failed to activate');
+            }
+            await (0, utils_1.sleepFor)(enums_1.HMSH_QUORUM_DELAY_MS * 2);
+            return await this.verifyWorkflowActive(hotMesh, appId, count + 1);
+        }
+        return true;
+    }
     async activateWorkflow(hotMesh, appId = factory_1.APP_ID, version = factory_1.APP_VERSION) {
         const app = await hotMesh.engine.store.getApp(appId);
         const appVersion = app?.version;
@@ -203,5 +216,6 @@ class ClientService {
         }
     }
 }
+ClientService.topics = [];
 ClientService.instances = new Map();
 exports.ClientService = ClientService;

package/build/services/durable/meshos.js

@@ -194,6 +194,9 @@ class MeshOSService {
         return await this.find(options.options ?? {}, ...args);
     }
     static generateSearchQuery(query) {
+        if (!Array.isArray(query) || query.length === 0) {
+            return '*';
+        }
         const my = new this();
         let queryString = query.map(q => {
             const { field, is, value, type } = q;

package/build/services/quorum/index.d.ts

@@ -3,8 +3,8 @@ import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
-import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
 import { HotMeshConfig } from '../../types/hotmesh';
+import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
@@ -30,6 +30,9 @@ declare class QuorumService {
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
     rollCall(delay?: number): Promise<QuorumProfile[]>;
-    activate(version: string, delay?: number): Promise<boolean>;
+    /**
+     * request a quorum; if successful activate the app version
+     */
+    activate(version: string, delay?: number, count?: number): Promise<boolean>;
 }
 export { QuorumService };

package/build/services/quorum/index.js

@@ -1,15 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.QuorumService = void 0;
-const key_1 = require("../../modules/key");
+const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const compiler_1 = require("../compiler");
 const redis_1 = require("../store/clients/redis");
 const ioredis_1 = require("../store/clients/ioredis");
 const ioredis_2 = require("../sub/clients/ioredis");
 const redis_2 = require("../sub/clients/redis");
-//wait time to see if quorum is reached
-const QUORUM_DELAY = 250;
+const hotmesh_1 = require("../../types/hotmesh");
 class QuorumService {
     constructor() {
         this.profiles = [];
@@ -30,8 +29,10 @@ class QuorumService {
             //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
             await instance.initStoreChannel(config.engine.store);
             await instance.initSubChannel(config.engine.sub);
-            await instance.subscribe.subscribe(key_1.KeyType.QUORUM, instance.subscriptionHandler(), appId); //general quorum subscription
-            await instance.subscribe.subscribe(key_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid); //app-specific quorum subscription (used for pubsub one-time request/response)
+            //general quorum subscription
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
+            //app-specific quorum subscription (used for pubsub one-time request/response)
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
             instance.engine.processWebHooks();
             instance.engine.processTimeHooks();
             return instance;
@@ -102,20 +103,20 @@ class QuorumService {
                 engine_id: this.guid,
                 namespace: this.namespace,
                 app_id: this.appId,
-                stream: this.engine.stream.mintKey(key_1.KeyType.STREAMS, { appId: this.appId })
+                stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId })
             };
         }
-        this.store.publish(key_1.KeyType.QUORUM, {
+        this.store.publish(hotmesh_1.KeyType.QUORUM, {
             type: 'pong',
             guid, originator,
             profile,
         }, appId);
     }
-    async requestQuorum(delay = QUORUM_DELAY, details = false) {
+    async requestQuorum(delay = enums_1.HMSH_QUORUM_DELAY_MS, details = false) {
         const quorum = this.quorum;
         this.quorum = 0;
         this.profiles.length = 0;
-        await this.store.publish(key_1.KeyType.QUORUM, {
+        await this.store.publish(hotmesh_1.KeyType.QUORUM, {
             type: 'ping',
             originator: this.guid,
             details,
@@ -126,7 +127,7 @@ class QuorumService {
     // ************* PUB/SUB METHODS *************
     //publish a message to the quorum
     async pub(quorumMessage) {
-        return await this.store.publish(key_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
+        return await this.store.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
     }
     //subscribe user to quorum messages
     async sub(callback) {
@@ -139,7 +140,7 @@ class QuorumService {
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
     // ************* COMPILER METHODS *************
-    async rollCall(delay = QUORUM_DELAY) {
+    async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
         await this.requestQuorum(delay, true);
         const targetStreams = [];
         const multi = this.store.getMulti();
@@ -160,18 +161,29 @@ class QuorumService {
         });
         return this.profiles;
     }
-    async activate(version, delay = QUORUM_DELAY) {
+    /**
+     * request a quorum; if successful activate the app version
+     */
+    async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
         version = version.toString();
+        const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
+        if (!canActivate) {
+            //another engine is already activating the app version
+            this.logger.debug('quorum-activation-awaiting', { version });
+            await (0, utils_1.sleepFor)(delay * 6);
+            const app = await this.store.getApp(this.appId, true);
+            return app?.active == true && app?.version === version;
+        }
         const config = await this.engine.getVID();
-        //request a quorum to activate the version
         await this.requestQuorum(delay);
         const q1 = await this.requestQuorum(delay);
         const q2 = await this.requestQuorum(delay);
         const q3 = await this.requestQuorum(delay);
         if (q1 && q1 === q2 && q2 === q3) {
             this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
-            this.store.publish(key_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
+            this.store.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
             await new Promise(resolve => setTimeout(resolve, delay));
+            await this.store.releaseScoutRole('activate');
             //confirm we received the activation message
             if (this.engine.untilVersion === version) {
                 this.logger.info('quorum-activation-succeeded', { version });
@@ -185,7 +197,12 @@ class QuorumService {
             }
         }
         else {
-            this.logger.info('quorum-rollcall-error', { q1, q2, q3 });
+            this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
+            this.store.releaseScoutRole('activate');
+            if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
+                //increase the delay (give the quorum time to respond) and try again
+                return await this.activate(version, delay * 2, count + 1);
+            }
             throw new Error(`Quorum not reached. Version ${version} not activated.`);
         }
     }

package/build/services/store/clients/redis.js

@@ -6,6 +6,7 @@ class RedisStoreService extends index_1.StoreService {
     constructor(redisClient) {
         super(redisClient);
         this.commands = {
+            set: 'SET',
             setnx: 'SETNX',
             del: 'DEL',
             expire: 'EXPIRE',

package/build/services/store/index.d.ts

@@ -46,7 +46,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * check for and process work items in the
      * time and signal task queues.
      */
-    reserveScoutRole(scoutType: 'time' | 'signal', delay?: number): Promise<boolean>;
+    reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay?: number): Promise<boolean>;
+    releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean>;
     getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
     setSettings(manifest: HotMeshSettings): Promise<any>;
     reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;

package/build/services/store/index.js

@@ -33,6 +33,7 @@ const errors_1 = require("../../modules/errors");
 class StoreService {
     constructor(redisClient) {
         this.commands = {
+            set: 'set',
             setnx: 'setnx',
             del: 'del',
             expire: 'expire',
@@ -106,12 +107,13 @@ class StoreService {
      */
     async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
         const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
-        const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`);
-        if (this.isSuccessful(success)) {
-            await this.redisClient[this.commands.expire](key, delay - 1);
-            return true;
-        }
-        return false;
+        const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
+        return this.isSuccessful(success);
+    }
+    async releaseScoutRole(scoutType) {
+        const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
+        const success = await this.exec('DEL', key);
+        return this.isSuccessful(success);
     }
     async getSettings(bCreate = false) {
         let settings = this.cache?.getSettings();

package/build/types/hotmesh.d.ts

@@ -42,7 +42,7 @@ type KeyStoreParams = {
     facet?: string;
     topic?: string;
     timeValue?: number;
-    scoutType?: 'signal' | 'time';
+    scoutType?: 'signal' | 'time' | 'activate';
 };
 type HotMesh = typeof HotMeshService;
 type RedisConfig = {

package/modules/enums.ts

@@ -22,6 +22,10 @@ export const HMSH_CODE_DURABLE_RETRYABLE = 599;
 
 export const HMSH_STATUS_UNKNOWN = 'unknown';
 
+// QUORUM
+export const HMSH_QUORUM_DELAY_MS = 250;
+export const HMSH_ACTIVATION_MAX_RETRY = 3;
+
 // ENGINE
 export const HMSH_OTT_WAIT_TIME = parseInt(process.env.HMSH_OTT_WAIT_TIME, 10) || 1000;
 export const HMSH_EXPIRE_JOB_SECONDS = parseInt(process.env.HMSH_EXPIRE_JOB_SECONDS, 10) || 1;

package/modules/utils.ts

@@ -10,7 +10,7 @@ export async function sleepFor(ms: number) {
 }
 
 export function guid(): string {
-  return nanoid();
+  return nanoid().replace(/[_-]/g, '0');
 }
 
 export function deterministicRandom(seed: number): number {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.42",
+  "version": "0.0.43",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/services/durable/client.ts

@@ -11,13 +11,14 @@ import { JobState } from '../../types/job';
 import { KeyService, KeyType } from '../../modules/key';
 import { Search } from './search';
 import { StreamStatus } from '../../types';
-import { HMSH_LOGLEVEL, HMSH_EXPIRE_JOB_SECONDS } from '../../modules/enums';
+import { HMSH_LOGLEVEL, HMSH_EXPIRE_JOB_SECONDS, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
+import { sleepFor } from '../../modules/utils';
 
 export class ClientService {
 
   connection: Connection;
-  topics: string[] = [];
   options: WorkflowOptions;
+  static topics: string[] = [];
   static instances = new Map<string, HotMesh | Promise<HotMesh>>();
 
   constructor(config: ClientConfig) {
@@ -29,8 +30,9 @@ export class ClientService {
     const instanceId = 'SINGLETON';
     if (ClientService.instances.has(instanceId)) {
       const hotMeshClient = await ClientService.instances.get(instanceId);
-      if (!this.topics.includes(workflowTopic)) {
-        this.topics.push(workflowTopic);
+      await this.verifyWorkflowActive(hotMeshClient, namespace ?? APP_ID);
+      if (!ClientService.topics.includes(workflowTopic)) {
+        ClientService.topics.push(workflowTopic);
         await this.createStream(hotMeshClient, workflowTopic, namespace);
       }
       return hotMeshClient;
@@ -196,6 +198,19 @@ export class ClientService {
     }
   }
 
+  async verifyWorkflowActive(hotMesh: HotMesh, appId = APP_ID, count = 0): Promise<boolean> {
+    const app = await hotMesh.engine.store.getApp(appId);
+    const appVersion = app?.version as unknown as number;
+    if(isNaN(appVersion)) {
+      if (count > 10) {
+        throw new Error('Workflow failed to activate');
+      }
+      await sleepFor(HMSH_QUORUM_DELAY_MS * 2);
+      return await this.verifyWorkflowActive(hotMesh, appId, count + 1);
+    }
+    return true;
+  }
+
   async activateWorkflow(hotMesh: HotMesh, appId = APP_ID, version = APP_VERSION): Promise<void> {
     const app = await hotMesh.engine.store.getApp(appId);
     const appVersion = app?.version as unknown as number;

package/services/durable/meshos.ts

@@ -295,6 +295,9 @@ export class MeshOSService {
   }
 
   static generateSearchQuery(query: FindWhereQuery[]) {
+    if (!Array.isArray(query) || query.length === 0) {
+      return '*';
+    }
     const my = new this();
     let queryString = query.map(q => {
       const { field, is, value, type } = q;

package/services/quorum/index.ts

@@ -1,4 +1,4 @@
-import { KeyType } from '../../modules/key';
+import { HMSH_ACTIVATION_MAX_RETRY, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
 import { identifyRedisType, sleepFor } from '../../modules/utils';
 import { CompilerService } from '../compiler';
 import { EngineService } from '../engine';
@@ -10,6 +10,7 @@ import { SubService } from '../sub';
 import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
 import { RedisSubService as RedisSub } from '../sub/clients/redis';
 import { CacheMode } from '../../types/cache';
+import { HotMeshConfig, KeyType } from '../../types/hotmesh';
 import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
 import {
   QuorumMessage,
@@ -18,13 +19,9 @@ import {
   SubscriptionCallback,
   ThrottleMessage
 } from '../../types/quorum';
-import { HotMeshApps, HotMeshConfig } from '../../types/hotmesh';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { RedisClientType } from '../../types/redisclient';
 
-//wait time to see if quorum is reached
-const QUORUM_DELAY = 250;
-
 class QuorumService {
   namespace: string;
   appId: string;
@@ -59,8 +56,18 @@ class QuorumService {
       //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
       await instance.initStoreChannel(config.engine.store);
       await instance.initSubChannel(config.engine.sub);
-      await instance.subscribe.subscribe(KeyType.QUORUM, instance.subscriptionHandler(), appId); //general quorum subscription
-      await instance.subscribe.subscribe(KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid); //app-specific quorum subscription (used for pubsub one-time request/response)
+      //general quorum subscription
+      await instance.subscribe.subscribe(
+        KeyType.QUORUM,
+        instance.subscriptionHandler(),
+        appId
+      );
+      //app-specific quorum subscription (used for pubsub one-time request/response)
+      await instance.subscribe.subscribe(
+        KeyType.QUORUM,
+        instance.subscriptionHandler(),
+        appId, instance.guid
+      );
       
       instance.engine.processWebHooks();
       instance.engine.processTimeHooks();
@@ -152,7 +159,7 @@ class QuorumService {
     );
   }
 
-  async requestQuorum(delay = QUORUM_DELAY, details = false): Promise<number> {
+  async requestQuorum(delay = HMSH_QUORUM_DELAY_MS, details = false): Promise<number> {
     const quorum = this.quorum;
     this.quorum = 0;
     this.profiles.length = 0;
@@ -188,7 +195,7 @@ class QuorumService {
 
 
   // ************* COMPILER METHODS *************
-  async rollCall(delay = QUORUM_DELAY): Promise<QuorumProfile[]> {
+  async rollCall(delay = HMSH_QUORUM_DELAY_MS): Promise<QuorumProfile[]> {
     await this.requestQuorum(delay, true);
     const targetStreams = [];
     const multi = this.store.getMulti();
@@ -209,10 +216,20 @@ class QuorumService {
     });
     return this.profiles;
   }
-  async activate(version: string, delay = QUORUM_DELAY): Promise<boolean> {
+  /**
+   * request a quorum; if successful activate the app version
+   */
+  async activate(version: string, delay = HMSH_QUORUM_DELAY_MS, count = 0): Promise<boolean> {
     version = version.toString();
+    const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
+    if (!canActivate) {
+      //another engine is already activating the app version
+      this.logger.debug('quorum-activation-awaiting', { version });
+      await sleepFor(delay * 6);
+      const app = await this.store.getApp(this.appId, true);
+      return app?.active == true && app?.version === version;
+    }
     const config = await this.engine.getVID();
-    //request a quorum to activate the version
     await this.requestQuorum(delay);
     const q1 = await this.requestQuorum(delay);
     const q2 = await this.requestQuorum(delay);
@@ -225,6 +242,7 @@ class QuorumService {
         this.appId
       );
       await new Promise(resolve => setTimeout(resolve, delay));
+      await this.store.releaseScoutRole('activate');
       //confirm we received the activation message
       if (this.engine.untilVersion === version) {
         this.logger.info('quorum-activation-succeeded', { version });
@@ -236,7 +254,12 @@ class QuorumService {
         throw new Error(`UntilVersion Not Received. Version ${version} not activated`);
       }
     } else {
-      this.logger.info('quorum-rollcall-error', { q1, q2, q3 });
+      this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
+      this.store.releaseScoutRole('activate');
+      if (count < HMSH_ACTIVATION_MAX_RETRY) {
+        //increase the delay (give the quorum time to respond) and try again
+        return await this.activate(version, delay * 2, count + 1);
+      }
       throw new Error(`Quorum not reached. Version ${version} not activated.`);
     }
   }

package/services/store/clients/redis.ts

@@ -18,6 +18,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
   constructor(redisClient: RedisClientType) {
     super(redisClient);
     this.commands = {
+      set: 'SET',
       setnx: 'SETNX',
       del: 'DEL',
       expire: 'EXPIRE',

package/services/store/index.ts

@@ -45,6 +45,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
   appId: string
   logger: ILogger;
   commands: Record<string, string> = {
+    set: 'set',
     setnx: 'setnx',
     del: 'del',
     expire: 'expire',
@@ -178,14 +179,16 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * check for and process work items in the
    * time and signal task queues.
    */
-  async reserveScoutRole(scoutType: 'time' | 'signal', delay = HMSH_SCOUT_INTERVAL_SECONDS): Promise<boolean> {
+  async reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay = HMSH_SCOUT_INTERVAL_SECONDS): Promise<boolean> {
     const key = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
-    const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${formatISODate(new Date())}`);
-    if (this.isSuccessful(success)) {
-      await this.redisClient[this.commands.expire](key, delay - 1);
-      return true;
-    }
-    return false;
+    const success = await this.exec('SET', key, `${scoutType}:${formatISODate(new Date())}`, 'NX', 'EX', `${delay - 1}`);
+    return this.isSuccessful(success);
+  }
+
+  async releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean> {
+    const key = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
+    const success = await this.exec('DEL', key);
+    return this.isSuccessful(success);
   }
 
   async getSettings(bCreate = false): Promise<HotMeshSettings> {

package/types/hotmesh.ts

@@ -44,7 +44,7 @@ type KeyStoreParams = {
   facet?: string;       //data path starting at root with values separated by colons (e.g. "object/type:bar")
   topic?: string;       //topic name (e.g., "foo" or "" for top-level)
   timeValue?: number;   //time value (rounded to minute) (for delete range)
-  scoutType?: 'signal' | 'time'; //a single member of the quorum serves as the 'scout' for the group, triaging tasks for the collective
+  scoutType?: 'signal' | 'time' | 'activate'; //a single member of the quorum serves as the 'scout' for the group, triaging tasks for the collective
 };
 
 type HotMesh = typeof HotMeshService;