@hotmeshio/hotmesh 0.0.42 → 0.0.44

package/build/services/durable/factory.js

@@ -60,6 +60,7 @@ const getWorkflowYAML = (app, version) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
           job:
             maps:
               done: false
@@ -263,10 +264,7 @@ const getWorkflowYAML = (app, version) => {
                   description: index will be appended later
             maps:
               signals: '{sigw1.output.data.signals}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-w']
-                  - ['{@string.concat}']
+              parentWorkflowId: '{$job.metadata.jid}'
               originJobId:
                 '@pipe':
                   - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
@@ -356,10 +354,7 @@ const getWorkflowYAML = (app, version) => {
                   description: index will be appended later
             maps:
               signals: '{w1.output.data.signals}'
-              parentWorkflowId:
-                '@pipe':
-                  - ['{$job.metadata.jid}', '-w']
-                  - ['{@string.concat}']
+              parentWorkflowId: '{$job.metadata.jid}'
               originJobId:
                 '@pipe':
                   - ['{t1.output.data.originJobId}', '{t1.output.data.originJobId}', '{$job.metadata.jid}']
@@ -526,6 +521,7 @@ const getWorkflowYAML = (app, version) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         w1a:
           title: Activity Worker - Calls Activity Functions
@@ -566,61 +562,6 @@ const getWorkflowYAML = (app, version) => {
         t1a:
           - to: w1a
 
-    - subscribes: ${app}.sleep.execute
-      publishes: ${app}.sleep.executed
-
-      expire: 0
-
-      input:
-        schema:
-          type: object
-          properties:
-            parentWorkflowId:
-              type: string
-            originJobId:
-              type: string
-            workflowId:
-              type: string
-            duration:
-              type: number
-              description: in seconds
-            index:
-              type: number
-      output:
-        schema:
-          type: object
-          properties:
-            done:
-              type: boolean
-            duration:
-              type: number
-            index:
-              type: number
-
-      activities:
-        t1s:
-          title: Sleep Flow Trigger
-          type: trigger
-          stats:
-            id: '{$self.input.data.workflowId}'
-            key: '{$self.input.data.parentWorkflowId}'
-            parent: '{$self.input.data.originJobId}'
-
-        a1s:
-          title: Sleep for a duration
-          type: hook
-          sleep: '{t1s.output.data.duration}'
-          job:
-            maps:
-              done: true
-              duration: '{t1s.output.data.duration}'
-              index: '{t1s.output.data.index}'
-              workflowId: '{t1s.output.data.workflowId}'
-
-      transitions:
-        t1s:
-          - to: a1s
-
     - subscribes: ${app}.wfsc.execute
       publishes: ${app}.wfsc.executed
 
@@ -665,6 +606,7 @@ const getWorkflowYAML = (app, version) => {
           stats:
             id: '{$self.input.data.cycleWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         a1wc:
           title: Pivot - All Cycling Descendants Point Here
@@ -836,6 +778,7 @@ const getWorkflowYAML = (app, version) => {
             id: '{$self.input.data.workflowId}'
             key: '{$self.input.data.parentWorkflowId}'
             parent: '{$self.input.data.originJobId}'
+            adjacent: '{$self.input.data.parentWorkflowId}'
 
         a1ww:
           title: WFS - signal entry point

package/build/services/durable/handle.d.ts

@@ -1,10 +1,14 @@
+import { ExporterService } from './exporter';
 import { HotMeshService as HotMesh } from '../hotmesh';
+import { DurableJobExport } from '../../types/exporter';
 import { JobInterruptOptions } from '../../types/job';
 export declare class WorkflowHandleService {
+    exporter: ExporterService;
     hotMesh: HotMesh;
     workflowTopic: string;
     workflowId: string;
     constructor(hotMesh: HotMesh, workflowTopic: string, workflowId: string);
+    export(): Promise<DurableJobExport>;
     /**
      * Sends a signal to the workflow. This is a way to send
      * a message to a workflow that is paused due to having

package/build/services/durable/handle.js

@@ -2,11 +2,16 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkflowHandleService = void 0;
 const enums_1 = require("../../modules/enums");
+const exporter_1 = require("./exporter");
 class WorkflowHandleService {
     constructor(hotMesh, workflowTopic, workflowId) {
         this.workflowTopic = workflowTopic;
         this.workflowId = workflowId;
         this.hotMesh = hotMesh;
+        this.exporter = new exporter_1.ExporterService(this.hotMesh.appId, this.hotMesh.engine.store, this.hotMesh.engine.logger);
+    }
+    async export() {
+        return this.exporter.export(this.workflowId);
     }
     /**
      * Sends a signal to the workflow. This is a way to send

package/build/services/durable/meshos.js

@@ -194,6 +194,9 @@ class MeshOSService {
         return await this.find(options.options ?? {}, ...args);
     }
     static generateSearchQuery(query) {
+        if (!Array.isArray(query) || query.length === 0) {
+            return '*';
+        }
         const my = new this();
         let queryString = query.map(q => {
             const { field, is, value, type } = q;

package/build/services/durable/workflow.js

@@ -37,7 +37,7 @@ class WorkflowService {
         const entityOrEmptyString = options.entity ?? '';
         //If the workflowId is not provided, it is generated from the entity and the workflow name
         const childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
-        const parentWorkflowId = `${workflowId}-f`;
+        const parentWorkflowId = workflowId;
         const client = new client_1.ClientService({
             connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
         });
@@ -75,32 +75,35 @@ class WorkflowService {
         const workflowSpan = store.get('workflowSpan');
         const COUNTER = store.get('counter');
         const execIndex = COUNTER.counter = COUNTER.counter + 1;
+        const sessionId = `-start${workflowDimension}-${execIndex}-`;
         //NOTE: this is the hash prefix; necessary for the search index to locate the entity
         const entityOrEmptyString = options.entity ?? '';
         //If the workflowId is not provided, it is generated from the entity and the workflow name
-        const childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
-        const parentWorkflowId = `${workflowId}-f`;
+        const parentWorkflowId = workflowId;
         const workflowTopic = `${options.entity ?? options.taskQueue}-${options.entity ?? options.workflowName}`;
-        try {
-            //get the status; if there is no error, return childJobId (what was spawned)
-            const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
-            await hotMeshClient.getStatus(childJobId);
+        const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
+        const keyParams = { appId: hotMeshClient.appId, jobId: workflowId };
+        const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
+        let childJobId = await hotMeshClient.engine.store.exec('HGET', workflowGuid, sessionId);
+        if (childJobId) {
             return childJobId;
         }
-        catch (error) {
-            const client = new client_1.ClientService({
-                connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
-            });
-            await client.workflow.start({
-                ...options,
-                namespace,
-                workflowId: childJobId,
-                parentWorkflowId,
-                workflowTrace,
-                workflowSpan,
-            });
-            return childJobId;
+        else {
+            childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
         }
+        const client = new client_1.ClientService({
+            connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
+        });
+        await client.workflow.start({
+            ...options,
+            namespace,
+            workflowId: childJobId,
+            parentWorkflowId,
+            workflowTrace,
+            workflowSpan,
+        });
+        await hotMeshClient.engine.store.exec('HSET', workflowGuid, sessionId, childJobId);
+        return childJobId;
     }
     /**
      * Wraps activities in a proxy that will durably run them
@@ -439,7 +442,7 @@ class WorkflowService {
                     arguments: Array.from(arguments),
                     //when the origin job is removed
                     originJobId: originJobId ?? workflowId,
-                    parentWorkflowId: `${workflowId}-a`,
+                    parentWorkflowId: workflowId,
                     workflowId: activityJobId,
                     workflowTopic: activityTopic,
                     activityName,

package/build/services/engine/index.d.ts

@@ -5,6 +5,7 @@ import { Interrupt } from '../activities/interrupt';
 import { Signal } from '../activities/signal';
 import { Worker } from '../activities/worker';
 import { Trigger } from '../activities/trigger';
+import { ExporterService } from '../exporter';
 import { ILogger } from '../logger';
 import { Router } from '../router';
 import { StoreService } from '../store';
@@ -18,15 +19,17 @@ import { JobState, JobData, JobMetadata, JobOutput, JobStatus, JobInterruptOptio
 import { HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshSettings } from '../../types/hotmesh';
 import { JobMessageCallback } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
-import { StringAnyType } from '../../types/serializer';
+import { StringAnyType, StringStringType } from '../../types/serializer';
 import { GetStatsOptions, IdsResponse, JobStatsInput, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamError, StreamStatus } from '../../types/stream';
 import { WorkListTaskType } from '../../types/task';
+import { JobExport } from '../../types/exporter';
 declare class EngineService {
     namespace: string;
     apps: HotMeshApps | null;
     appId: string;
     guid: string;
+    exporter: ExporterService | null;
     router: Router | null;
     store: StoreService<RedisClient, RedisMulti> | null;
     stream: StreamService<RedisClient, RedisMulti> | null;
@@ -88,6 +91,8 @@ declare class EngineService {
      * it will be expired immediately.
      */
     resolveExpires(context: JobState, options: JobCompletionOptions): number;
+    export(jobId: string): Promise<JobExport>;
+    getRaw(jobId: string): Promise<StringStringType>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;
     getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;

package/build/services/engine/index.js

@@ -9,6 +9,7 @@ const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const activities_1 = __importDefault(require("../activities"));
 const compiler_1 = require("../compiler");
+const exporter_1 = require("../exporter");
 const reporter_1 = require("../reporter");
 const router_1 = require("../router");
 const serializer_1 = require("../serializer");
@@ -41,8 +42,8 @@ class EngineService {
             await instance.initStreamChannel(config.engine.stream);
             instance.router = instance.initRouter(config);
             instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
-            //the task service is used by the engine to process `webhooks` and `timehooks`
             instance.taskService = new task_1.TaskService(instance.store, logger);
+            instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
             return instance;
         }
     }
@@ -526,9 +527,15 @@ class EngineService {
         return options.expire ?? context.metadata.expire ?? enums_1.HMSH_EXPIRE_JOB_SECONDS;
     }
     // ****** GET JOB STATE/COLLATION STATUS BY ID *********
+    async export(jobId) {
+        return await this.exporter.export(jobId);
+    }
+    async getRaw(jobId) {
+        return await this.store.getRaw(jobId);
+    }
     async getStatus(jobId) {
         const { id: appId } = await this.getVID();
-        return this.store.getStatus(jobId, appId);
+        return await this.store.getStatus(jobId, appId);
     }
     //todo: add 'options' parameter;
     //      (e.g, if {dimensions:true}, use hscan to deliver

package/build/services/exporter/index.d.ts

@@ -0,0 +1,46 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { StringStringType, Symbols } from "../../types/serializer";
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { DependencyExport, ExportOptions, JobActionExport, JobExport } from '../../types/exporter';
+import { SerializerService } from '../serializer';
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Expands process data and includes dependency list
+ */
+declare class ExporterService {
+    appId: string;
+    logger: ILogger;
+    serializer: SerializerService;
+    store: StoreService<RedisClient, RedisMulti>;
+    symbols: Promise<Symbols> | Symbols;
+    constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    /**
+     * Convert the job hash and dependency list into a JobExport object.
+     * This object contains various facets that describe the interaction
+     * in terms relevant to narrative storytelling.
+     */
+    export(jobId: string, options?: ExportOptions): Promise<JobExport>;
+    /**
+     * Inflates the key from Redis, 3-character symbol
+     * into a human-readable JSON path, reflecting the
+     * tree-like structure of the unidimensional Hash
+     */
+    inflateKey(key: string): string;
+    /**
+     * Inflates the job data from Redis into a JobExport object
+     * @param jobHash - the job data from Redis
+     * @param dependencyList - the list of dependencies for the job
+     * @returns - the inflated job data
+     */
+    inflate(jobHash: StringStringType, dependencyList: string[]): JobExport;
+    /**
+     * Inflates the dependency data from Redis into a JobExport object by
+     * organizing the dimensional isolate in sch a way asto interleave
+     * into a story
+     * @param data - the dependency data from Redis
+     * @returns - the organized dependency data
+     */
+    inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
+}
+export { ExporterService };

package/build/services/exporter/index.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExporterService = void 0;
+const serializer_1 = require("../serializer");
+const utils_1 = require("../../modules/utils");
+/**
+ * Downloads job data from Redis (hscan, hmget, hgetall)
+ * Expands process data and includes dependency list
+ */
+class ExporterService {
+    constructor(appId, store, logger) {
+        this.appId = appId;
+        this.logger = logger;
+        this.store = store;
+        this.serializer = new serializer_1.SerializerService();
+    }
+    /**
+     * Convert the job hash and dependency list into a JobExport object.
+     * This object contains various facets that describe the interaction
+     * in terms relevant to narrative storytelling.
+     */
+    async export(jobId, options = {}) {
+        if (!this.symbols) {
+            this.symbols = this.store.getAllSymbols();
+            this.symbols = await this.symbols;
+        }
+        const depData = await this.store.getDependencies(jobId);
+        const jobData = await this.store.getRaw(jobId);
+        const jobExport = this.inflate(jobData, depData);
+        return jobExport;
+    }
+    /**
+     * Inflates the key from Redis, 3-character symbol
+     * into a human-readable JSON path, reflecting the
+     * tree-like structure of the unidimensional Hash
+     */
+    inflateKey(key) {
+        return (key in this.symbols) ? this.symbols[key] : key;
+    }
+    /**
+     * Inflates the job data from Redis into a JobExport object
+     * @param jobHash - the job data from Redis
+     * @param dependencyList - the list of dependencies for the job
+     * @returns - the inflated job data
+     */
+    inflate(jobHash, dependencyList) {
+        //the list of actions taken in the workflow and hook functions
+        const actions = {
+            hooks: {},
+            main: {
+                cursor: -1,
+                items: []
+            }
+        };
+        const process = {};
+        const dependencies = this.inflateDependencyData(dependencyList, actions);
+        const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
+        Object.entries(jobHash).forEach(([key, value]) => {
+            const match = key.match(regex);
+            if (match) {
+                //activity process state
+                const [_, letters, numbers] = match;
+                const path = this.inflateKey(letters);
+                const dimensions = `${numbers.replace(/,/g, '/')}`;
+                const resolved = this.serializer.fromString(value);
+                process[`${dimensions}/${path}`] = resolved;
+            }
+            else if (key.length === 3) {
+                //job state
+                process[this.inflateKey(key)] = this.serializer.fromString(value);
+            }
+        });
+        return {
+            dependencies,
+            process: (0, utils_1.restoreHierarchy)(process),
+            status: jobHash[':'],
+        };
+    }
+    /**
+     * Inflates the dependency data from Redis into a JobExport object by
+     * organizing the dimensional isolate in sch a way asto interleave
+     * into a story
+     * @param data - the dependency data from Redis
+     * @returns - the organized dependency data
+     */
+    inflateDependencyData(data, actions) {
+        const hookReg = /([0-9,]+)-(\d+)$/;
+        const flowReg = /-(\d+)$/;
+        return data.map((dependency, index) => {
+            const [action, topic, gid, ...jid] = dependency.split('::');
+            const jobId = jid.join('::');
+            const match = jobId.match(hookReg);
+            let prefix;
+            let type;
+            let dimensionKey = '';
+            if (match) {
+                //hook-originating dependency
+                const [_, dimension, counter] = match;
+                dimensionKey = dimension.split(',').join('/');
+                prefix = `${dimensionKey}[${counter}]`;
+                type = 'hook';
+            }
+            else {
+                const match = jobId.match(flowReg);
+                if (match) {
+                    //main workflow-originating dependency
+                    const [_, counter] = match;
+                    prefix = `[${counter}]`;
+                    type = 'flow';
+                }
+                else {
+                    //'other' types like signal cleanup
+                    prefix = '/';
+                    type = 'other';
+                }
+            }
+            return {
+                type: action,
+                topic,
+                gid,
+                jid: jobId,
+            };
+        });
+    }
+}
+exports.ExporterService = ExporterService;

package/build/services/hotmesh/index.d.ts

@@ -7,7 +7,8 @@ import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
 import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
 import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
 import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
-import { StringAnyType } from '../../types/serializer';
+import { StringAnyType, StringStringType } from '../../types/serializer';
+import { JobExport } from '../../types/exporter';
 declare class HotMeshService {
     namespace: string;
     appId: string;
@@ -35,6 +36,8 @@ declare class HotMeshService {
     plan(path: string): Promise<HotMeshManifest>;
     deploy(pathOrYAML: string): Promise<HotMeshManifest>;
     activate(version: string, delay?: number): Promise<boolean>;
+    export(jobId: string): Promise<JobExport>;
+    getRaw(jobId: string): Promise<StringStringType>;
     getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
     getStatus(jobId: string): Promise<JobStatus>;
     getState(topic: string, jobId: string): Promise<JobOutput>;

package/build/services/hotmesh/index.js

@@ -104,6 +104,12 @@ class HotMeshService {
         return await this.quorum?.activate(version, delay);
     }
     // ************* REPORTER METHODS *************
+    async export(jobId) {
+        return await this.engine?.export(jobId);
+    }
+    async getRaw(jobId) {
+        return await this.engine?.getRaw(jobId);
+    }
     async getStats(topic, query) {
         return await this.engine?.getStats(topic, query);
     }

package/build/services/quorum/index.d.ts

@@ -3,8 +3,8 @@ import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { SubService } from '../sub';
 import { CacheMode } from '../../types/cache';
-import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
 import { HotMeshConfig } from '../../types/hotmesh';
+import { QuorumMessageCallback, QuorumProfile, SubscriptionCallback, ThrottleMessage } from '../../types/quorum';
 import { RedisClient, RedisMulti } from '../../types/redis';
 declare class QuorumService {
     namespace: string;
@@ -30,6 +30,9 @@ declare class QuorumService {
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
     rollCall(delay?: number): Promise<QuorumProfile[]>;
-    activate(version: string, delay?: number): Promise<boolean>;
+    /**
+     * request a quorum; if successful activate the app version
+     */
+    activate(version: string, delay?: number, count?: number): Promise<boolean>;
 }
 export { QuorumService };

package/build/services/quorum/index.js

@@ -1,15 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.QuorumService = void 0;
-const key_1 = require("../../modules/key");
+const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const compiler_1 = require("../compiler");
 const redis_1 = require("../store/clients/redis");
 const ioredis_1 = require("../store/clients/ioredis");
 const ioredis_2 = require("../sub/clients/ioredis");
 const redis_2 = require("../sub/clients/redis");
-//wait time to see if quorum is reached
-const QUORUM_DELAY = 250;
+const hotmesh_1 = require("../../types/hotmesh");
 class QuorumService {
     constructor() {
         this.profiles = [];
@@ -30,8 +29,10 @@ class QuorumService {
             //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
             await instance.initStoreChannel(config.engine.store);
             await instance.initSubChannel(config.engine.sub);
-            await instance.subscribe.subscribe(key_1.KeyType.QUORUM, instance.subscriptionHandler(), appId); //general quorum subscription
-            await instance.subscribe.subscribe(key_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid); //app-specific quorum subscription (used for pubsub one-time request/response)
+            //general quorum subscription
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
+            //app-specific quorum subscription (used for pubsub one-time request/response)
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
             instance.engine.processWebHooks();
             instance.engine.processTimeHooks();
             return instance;
@@ -102,20 +103,21 @@ class QuorumService {
                 engine_id: this.guid,
                 namespace: this.namespace,
                 app_id: this.appId,
-                stream: this.engine.stream.mintKey(key_1.KeyType.STREAMS, { appId: this.appId })
+                stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId }),
+                counts: this.engine.router.counts,
             };
         }
-        this.store.publish(key_1.KeyType.QUORUM, {
+        this.store.publish(hotmesh_1.KeyType.QUORUM, {
             type: 'pong',
             guid, originator,
             profile,
         }, appId);
     }
-    async requestQuorum(delay = QUORUM_DELAY, details = false) {
+    async requestQuorum(delay = enums_1.HMSH_QUORUM_DELAY_MS, details = false) {
         const quorum = this.quorum;
         this.quorum = 0;
         this.profiles.length = 0;
-        await this.store.publish(key_1.KeyType.QUORUM, {
+        await this.store.publish(hotmesh_1.KeyType.QUORUM, {
             type: 'ping',
             originator: this.guid,
             details,
@@ -126,7 +128,7 @@ class QuorumService {
     // ************* PUB/SUB METHODS *************
     //publish a message to the quorum
     async pub(quorumMessage) {
-        return await this.store.publish(key_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
+        return await this.store.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
     }
     //subscribe user to quorum messages
     async sub(callback) {
@@ -139,7 +141,7 @@ class QuorumService {
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
     // ************* COMPILER METHODS *************
-    async rollCall(delay = QUORUM_DELAY) {
+    async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
         await this.requestQuorum(delay, true);
         const targetStreams = [];
         const multi = this.store.getMulti();
@@ -160,18 +162,29 @@ class QuorumService {
         });
         return this.profiles;
     }
-    async activate(version, delay = QUORUM_DELAY) {
+    /**
+     * request a quorum; if successful activate the app version
+     */
+    async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
         version = version.toString();
+        const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
+        if (!canActivate) {
+            //another engine is already activating the app version
+            this.logger.debug('quorum-activation-awaiting', { version });
+            await (0, utils_1.sleepFor)(delay * 6);
+            const app = await this.store.getApp(this.appId, true);
+            return app?.active == true && app?.version === version;
+        }
         const config = await this.engine.getVID();
-        //request a quorum to activate the version
         await this.requestQuorum(delay);
         const q1 = await this.requestQuorum(delay);
         const q2 = await this.requestQuorum(delay);
         const q3 = await this.requestQuorum(delay);
         if (q1 && q1 === q2 && q2 === q3) {
             this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
-            this.store.publish(key_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
+            this.store.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
             await new Promise(resolve => setTimeout(resolve, delay));
+            await this.store.releaseScoutRole('activate');
             //confirm we received the activation message
             if (this.engine.untilVersion === version) {
                 this.logger.info('quorum-activation-succeeded', { version });
@@ -185,7 +198,12 @@ class QuorumService {
             }
         }
         else {
-            this.logger.info('quorum-rollcall-error', { q1, q2, q3 });
+            this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
+            this.store.releaseScoutRole('activate');
+            if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
+                //increase the delay (give the quorum time to respond) and try again
+                return await this.activate(version, delay * 2, count + 1);
+            }
             throw new Error(`Quorum not reached. Version ${version} not activated.`);
         }
     }

package/build/services/router/index.d.ts

@@ -17,6 +17,9 @@ declare class Router {
     logger: ILogger;
     throttle: number;
     errorCount: number;
+    counts: {
+        [key: string]: number;
+    };
     currentTimerId: NodeJS.Timeout | null;
     shouldConsume: boolean;
     constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);