@hotmeshio/hotmesh 0.0.35 → 0.0.37

package/services/engine/index.ts

@@ -4,7 +4,7 @@ import {
   STATUS_CODE_SUCCESS,
   STATUS_CODE_PENDING,
   STATUS_CODE_TIMEOUT, 
-  DURABLE_EXPIRE_SECONDS} from '../../modules/enums';
+  DURABLE_EXPIRE_SECONDS } from '../../modules/enums';
 import {
   formatISODate,
   getSubscriptionTopic,
@@ -23,9 +23,8 @@ import { Trigger } from '../activities/trigger';
 import { CompilerService } from '../compiler';
 import { ILogger } from '../logger';
 import { ReporterService } from '../reporter';
+import { Router } from '../router';
 import { SerializerService } from '../serializer';
-import { StoreSignaler } from '../signaler/store';
-import { StreamSignaler } from '../signaler/stream';
 import { StoreService } from '../store';
 import { RedisStoreService as RedisStore } from '../store/clients/redis';
 import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
@@ -84,12 +83,11 @@ class EngineService {
   apps: HotMeshApps | null;
   appId: string;
   guid: string;
+  router: Router | null;
   store: StoreService<RedisClient, RedisMulti> | null;
   stream: StreamService<RedisClient, RedisMulti> | null;
   subscribe: SubService<RedisClient, RedisMulti> | null;
-  storeSignaler: StoreSignaler | null;
-  streamSignaler: StreamSignaler | null;
-  task: TaskService | null;
+  taskService: TaskService | null;
   logger: ILogger;
   cacheMode: CacheMode = 'cache';
   untilVersion: string | null = null;
@@ -110,9 +108,9 @@ class EngineService {
       await instance.initStoreChannel(config.engine.store);
       await instance.initSubChannel(config.engine.sub);
       await instance.initStreamChannel(config.engine.stream);
-      instance.streamSignaler = instance.initStreamSignaler(config);
+      instance.router = instance.initRouter(config);
 
-      instance.streamSignaler.consumeMessages(
+      instance.router.consumeMessages(
         instance.stream.mintKey(
           KeyType.STREAMS,
           { appId: instance.appId },
@@ -122,12 +120,8 @@ class EngineService {
         instance.processStreamMessage.bind(instance)
       );
 
-      //the storeSignaler service is used by the engine to create `webhooks`
-      //todo: unify/move to the task service (it manages all `signal` types)
-      instance.storeSignaler = new StoreSignaler(instance.store, logger);
-
       //the task service is used by the engine to process `webhooks` and `timehooks`
-      instance.task = new TaskService(instance.store, logger);
+      instance.taskService = new TaskService(instance.store, logger);
 
       return instance;
     }
@@ -181,8 +175,8 @@ class EngineService {
     );
   }
 
-  initStreamSignaler(config: HotMeshConfig): StreamSignaler {
-    return new StreamSignaler(
+  initRouter(config: HotMeshConfig): Router {
+    return new Router(
       {
         namespace: this.namespace,
         appId: this.appId,
@@ -231,21 +225,20 @@ class EngineService {
   }
 
   async processWebHooks() {
-    this.task.processWebHooks((this.hook).bind(this));
+    this.taskService.processWebHooks((this.hook).bind(this));
   }
 
   async processTimeHooks() {
-    this.task.processTimeHooks((this.hookTime).bind(this));
+    this.taskService.processTimeHooks((this.hookTime).bind(this));
   }
 
   async throttle(delayInMillis: number) {
-    this.streamSignaler.setThrottle(delayInMillis);
+    this.router.setThrottle(delayInMillis);
   }
 
   // ************* METADATA/MODEL METHODS *************
   async initActivity(topic: string, data: JobData = {}, context?: JobState): Promise<Await|Cycle|Hook|Signal|Trigger|Worker|Interrupt> {
     const [activityId, schema] = await this.getSchema(topic);
-    polyfill
     const ActivityHandler = Activities[polyfill.resolveActivityType(schema.type)];
     if (ActivityHandler) {
       const utc = formatISODate(new Date());
@@ -329,6 +322,7 @@ class EngineService {
   async processStreamMessage(streamData: StreamDataResponse): Promise<void> {
     this.logger.debug('engine-process-stream-message', {
       jid: streamData.metadata.jid,
+      gid: streamData.metadata.gid,
       dad: streamData.metadata.dad,
       aid: streamData.metadata.aid,
       status: streamData.status || StreamStatus.SUCCESS,
@@ -338,41 +332,83 @@ class EngineService {
     const context: PartialJobState = {
       metadata: {
         jid: streamData.metadata.jid,
+        gid: streamData.metadata.gid,
         dad: streamData.metadata.dad,
         aid: streamData.metadata.aid,
       },
       data: streamData.data,
     };
-    if (streamData.type === StreamDataType.TIMEHOOK || streamData.type === StreamDataType.WEBHOOK || streamData.type === StreamDataType.TRANSITION) {
-      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context as JobState) as Hook;
-      if (streamData.type === StreamDataType.TIMEHOOK) {
-        await activityHandler.processTimeHookEvent(streamData.metadata.jid);
-      } else if (streamData.type === StreamDataType.TRANSITION) {
-        await activityHandler.process();
-      } else {
-        //a 202 code keeps the hook alive (hooks are single-use by default)
-        await activityHandler.processWebHookEvent(streamData.status, streamData.code);
-      }
+    if (streamData.type === StreamDataType.TIMEHOOK) {
+      //TIMEHOOK AWAKEN
+      const activityHandler = await this.initActivity(
+        `.${streamData.metadata.aid}`,
+        context.data,
+        context as JobState,
+        ) as Hook;
+      await activityHandler.processTimeHookEvent(streamData.metadata.jid);
+    } else if (streamData.type === StreamDataType.WEBHOOK) {
+      //WEBHOOK AWAKEN (SIGNAL IN)
+      const activityHandler = await this.initActivity(
+        `.${streamData.metadata.aid}`,
+        context.data,
+        context as JobState,
+        ) as Hook;
+      await activityHandler.processWebHookEvent(
+        streamData.status,
+        streamData.code
+      );
+    } else if (streamData.type === StreamDataType.TRANSITION) {
+      //TRANSITION (ADJACENT ACTIVITY)
+      const activityHandler = await this.initActivity(
+        `.${streamData.metadata.aid}`,
+        context.data,
+        context as JobState,
+      ) as Hook; //todo: `as Activity` (type is more generic)
+      await activityHandler.process();
     } else if (streamData.type === StreamDataType.AWAIT) {
+      //TRIGGER JOB
       context.metadata = {
         ...context.metadata,
         pj: streamData.metadata.jid,
+        pg: streamData.metadata.gid,
         pd: streamData.metadata.dad,
         pa: streamData.metadata.aid,
         trc: streamData.metadata.trc,
         spn: streamData.metadata.spn,
        };
-      const activityHandler = await this.initActivity(streamData.metadata.topic, streamData.data, context as JobState) as Trigger;
+      const activityHandler = await this.initActivity(
+        streamData.metadata.topic,
+        streamData.data,
+        context as JobState
+      ) as Trigger;
       await activityHandler.process();
     } else if (streamData.type === StreamDataType.RESULT) {
-      const activityHandler = await this.initActivity(`.${context.metadata.aid}`, streamData.data, context as JobState) as Await;
-      await activityHandler.processEvent(streamData.status, streamData.code);
+      //AWAIT RESULT
+      const activityHandler = await this.initActivity(
+        `.${context.metadata.aid}`,
+        streamData.data,
+        context as JobState,
+      ) as Await;
+      await activityHandler.processEvent(
+        streamData.status,
+        streamData.code,
+      );
     } else {
-      const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, streamData.data, context as JobState) as Worker;
-      await activityHandler.processEvent(streamData.status, streamData.code, 'output');
+      //WORKER RESULT
+      const activityHandler = await this.initActivity(
+        `.${streamData.metadata.aid}`,
+        streamData.data,
+        context as JobState,
+      ) as Worker;
+      await activityHandler.processEvent(
+        streamData.status,
+        streamData.code,
+        'output'
+      );
     }
     this.logger.debug('engine-process-stream-message-end', {
       jid: streamData.metadata.jid,
+      gid: streamData.metadata.gid,
       aid: streamData.metadata.aid
     });
   }
@@ -387,6 +423,7 @@ class EngineService {
         metadata: {
           guid: guid(),
           jid: context.metadata.pj,
+          gid: context.metadata.pg,
           dad: context.metadata.pd,
           aid: context.metadata.pa,
           trc: context.metadata.trc,
@@ -406,7 +443,7 @@ class EngineService {
         streamData.status = StreamStatus.SUCCESS;
         streamData.code = STATUS_CODE_SUCCESS;
       }
-      return (await this.streamSignaler?.publishMessage(null, streamData)) as string;
+      return (await this.router?.publishMessage(null, streamData)) as string;
     }
   }
   hasParentJob(context: JobState): boolean {
@@ -437,7 +474,7 @@ class EngineService {
 
   // ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
   async hook(topic: string, data: JobData, status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<string> {
-    const hookRule = await this.storeSignaler.getHookRule(topic);
+    const hookRule = await this.taskService.getHookRule(topic);
     const [aid] = await this.getSchema(`.${hookRule.to}`);
     const streamData: StreamData = {
       type: StreamDataType.WEBHOOK,
@@ -450,9 +487,9 @@ class EngineService {
       },
       data,
     };
-    return await this.streamSignaler.publishMessage(null, streamData) as string;
+    return await this.router.publishMessage(null, streamData) as string;
   }
-  async hookTime(jobId: string, activityId: string, type?: 'sleep'|'expire'|'interrupt'): Promise<string | void> {
+  async hookTime(jobId: string, gId: string, activityId: string, type?: 'sleep'|'expire'|'interrupt'): Promise<string | void> {
     if (type === 'interrupt') {
       return await this.interrupt(
         activityId, //note: 'activityId' is the actually job topic
@@ -470,16 +507,17 @@ class EngineService {
       metadata: {
         guid: guid(),
         jid: jobId,
-        aid,
+        gid: gId,
         dad,
+        aid,
       },
       data: { timestamp: Date.now() },
     };
-    await this.streamSignaler.publishMessage(null, streamData);
+    await this.router.publishMessage(null, streamData);
   }
   async hookAll(hookTopic: string, data: JobData, keyResolver: JobStatsInput, queryFacets: string[] = []): Promise<string[]> {
     const config = await this.getVID();
-    const hookRule = await this.storeSignaler.getHookRule(hookTopic);
+    const hookRule = await this.taskService.getHookRule(hookTopic);
     if (hookRule) {
       const subscriptionTopic = await getSubscriptionTopic(hookRule.to, this.store, config)
       const resolvedQuery = await this.resolveQuery(subscriptionTopic, keyResolver);
@@ -597,7 +635,7 @@ class EngineService {
     }
   }
   async add(streamData: StreamData|StreamDataResponse): Promise<string> {
-    return await this.streamSignaler.publishMessage(null, streamData) as string;
+    return await this.router.publishMessage(null, streamData) as string;
   }
 
   registerJobCallback(jobId: string, jobCallback: JobMessageCallback) {
@@ -632,7 +670,7 @@ class EngineService {
       this.pubPermSubs(context, jobOutput, options.emit);
     }
     if (!options.emit) {
-      this.task.registerJobForCleanup(
+      this.taskService.registerJobForCleanup(
         context.metadata.jid,
         this.resolveExpires(context, options),
         options,
@@ -647,7 +685,7 @@ class EngineService {
    * it will be expired immediately.
    */
   resolveExpires(context: JobState, options: JobCompletionOptions): number {
-    return context.metadata.expire ?? options.expire ?? DURABLE_EXPIRE_SECONDS;
+    return options.expire ?? context.metadata.expire ?? DURABLE_EXPIRE_SECONDS;
   }
 
 

package/services/hotmesh/index.ts

@@ -4,8 +4,8 @@ import { RedisConnection } from '../connector/clients/redis';
 import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
 import { EngineService } from '../engine';
 import { LoggerService, ILogger } from '../logger';
-import { StreamSignaler } from '../signaler/stream';
 import { QuorumService } from '../quorum';
+import { Router } from '../router';
 import { WorkerService } from '../worker';
 import {
   JobState,
@@ -193,14 +193,14 @@ class HotMeshService {
   static async stop() {
     if (!this.disconnecting) {
       this.disconnecting = true;
-      await StreamSignaler.stopConsuming();
+      await Router.stopConsuming();
       await RedisConnection.disconnectAll();
       await IORedisConnection.disconnectAll();
     }
   }
 
   stop() {
-    this.engine?.task.cancelCleanup();
+    this.engine?.taskService.cancelCleanup();
   }
 
   async compress(terms: string[]): Promise<boolean> {

package/services/{signaler/stream.ts → router/index.ts}

@@ -27,8 +27,8 @@ import {
   StreamStatus
 } from '../../types/stream';
 
-class StreamSignaler {
-  static signalers: Set<StreamSignaler> = new Set();
+class Router {
+  static instances: Set<Router> = new Set();
   appId: string;
   guid: string;
   role: StreamRole;
@@ -70,7 +70,7 @@ class StreamSignaler {
 
   async consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse|void>): Promise<void> {
     this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
-    StreamSignaler.signalers.add(this);
+    Router.instances.add(this);
     this.shouldConsume = true;
     await this.createGroup(stream, group);
     let lastCheckedPendingMessagesAt = Date.now();
@@ -254,7 +254,7 @@ class StreamSignaler {
   }
   
   static async stopConsuming() {
-    for (const instance of [...StreamSignaler.signalers]) {
+    for (const instance of [...Router.instances]) {
       instance.stopConsuming();
     }
     await sleepFor(BLOCK_TIME_MS);
@@ -356,4 +356,4 @@ class StreamSignaler {
   }
 }
 
-export { StreamSignaler };
+export { Router };

package/services/serializer/index.ts

@@ -19,7 +19,7 @@ export const MDATA_SYMBOLS = {
     KEYS: ['au', 'err', 'l2s']
   },
   JOB: {
-    KEYS: ['ngn', 'tpc', 'pj', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+    KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
   },
   JOB_UPDATE: {
     KEYS: ['ju', 'err']

package/services/store/clients/ioredis.ts

@@ -32,6 +32,7 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
       }
       return response as string[];
     }
+    return response;
   }
 
   hGetAllResult(result: any) {

package/services/store/index.ts

@@ -29,7 +29,7 @@ import { Transitions } from '../../types/transition';
 import { formatISODate, getSymKey } from '../../modules/utils';
 import { ReclaimedMessageType } from '../../types/stream';
 import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
-import { STATUS_CODE_INTERRUPT } from '../../modules/enums';
+import { SCOUT_INTERVAL_SECONDS, STATUS_CODE_INTERRUPT } from '../../modules/enums';
 import { GetStateError } from '../../modules/errors';
 
 interface AbstractRedisClient {
@@ -168,10 +168,19 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     this.cache.invalidate();
   }
 
-  async reserveEngineId(engineId: string): Promise<boolean> {
-    const key = this.mintKey(KeyType.ENGINE_ID, { engineId });
-    const success = await this.redisClient[this.commands.setnx](key, 'id', 1);
-    return this.isSuccessful(success);
+  /**
+   * At any given time only a single engine will
+   * check for and process work items in the
+   * time and signal task queues.
+   */
+  async reserveScoutRole(scoutType: 'time' | 'signal', delay = SCOUT_INTERVAL_SECONDS): Promise<boolean> {
+    const key = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
+    const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${formatISODate(new Date())}`);
+    if (this.isSuccessful(success)) {
+      await this.redisClient[this.commands.expire](key, delay - 1);
+      return true;
+    }
+    return false;
   }
 
   async getSettings(bCreate = false): Promise<HotMeshSettings> {
@@ -380,11 +389,11 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * list (added via RPUSH) are LPOPed. If origin was expired, then
    * LPOPed items from the list are likewise expired;
    */
-  async setDependency(originJobId: string, topic: string, jobId: string, multi? : U): Promise<any> {
+  async setDependency(originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
     const depParams = { appId: this.appId, jobId: originJobId };
     const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
-    privateMulti[this.commands.rpush](depKey, `expire::${topic}::${jobId}`);
+    privateMulti[this.commands.rpush](depKey, `expire::${topic}::${gId}::${jobId}`);
     if (!multi) {
       return await privateMulti.exec();
     }
@@ -784,9 +793,9 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * for the given sleep group. Sleep groups are
    * organized into 'n'-second blocks (LISTS))
    */
-  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', deletionTime: number, multi?: U): Promise<void> {
+  async registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', deletionTime: number, multi?: U): Promise<void> {
     const listKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
-    const timeEvent = `${type}::${activityId}::${jobId}`
+    const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
     const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
     if (multi || len === 1) {
       const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
@@ -794,7 +803,8 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
-  async getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt'] | void> {
+  async getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt'] | boolean> {
+    const existing = Boolean(listKey);
     const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
     listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
     if (listKey) {
@@ -802,11 +812,12 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       const timeEvent = await this.redisClient[this.commands.lpop](pKey);
       if (timeEvent) {
         //there are 3 time-related event triggers: sleep, expire, interrupt
-        const [_type, activityId, ...jobId] = timeEvent.split('::');
-        return [listKey, jobId.join('::'), activityId, pType];
+        const [_type, activityId, gId, ...jobId] = timeEvent.split('::');
+        return [listKey, jobId.join('::'), gId, activityId, pType];
       }
       await this.redisClient[this.commands.zrem](zsetKey, listKey);
     }
+    return existing;
   }
 
   /**

package/services/task/index.ts

@@ -1,17 +1,20 @@
 import {
   EXPIRE_DURATION,
-  FIDELITY_SECONDS } from '../../modules/enums';
+  FIDELITY_SECONDS, 
+  SCOUT_INTERVAL_SECONDS} from '../../modules/enums';
 import { XSleepFor, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
-import { HookInterface } from '../../types/hook';
-import { JobCompletionOptions } from '../../types/job';
+import { HookInterface, HookRule, HookSignal } from '../../types/hook';
+import { JobCompletionOptions, JobState } from '../../types/job';
 import { RedisClient, RedisMulti } from '../../types/redis';
+import { Pipe } from '../pipe';
 
 class TaskService {
   store: StoreService<RedisClient, RedisMulti>;
   logger: ILogger;
   cleanupTimeout: NodeJS.Timeout | null = null;
+  isScout: boolean = false;
 
   constructor(
     store: StoreService<RedisClient, RedisMulti>,
@@ -50,28 +53,59 @@ class TaskService {
     }
   }
 
-  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', inSeconds = FIDELITY_SECONDS, multi?: RedisMulti): Promise<void> {
+  async registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', inSeconds = FIDELITY_SECONDS, multi?: RedisMulti): Promise<void> {
     const awakenTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
-    await this.store.registerTimeHook(jobId, activityId, type, awakenTimeSlot, multi);
+    await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, multi);
   }
 
-  async processTimeHooks(timeEventCallback: (jobId: string, activityId: string, type: 'sleep'|'expire'|'interrupt') => Promise<void>, listKey?: string): Promise<void> {
-    try {
-      const timeJob = await this.store.getNextTimeJob(listKey);
-      if (timeJob) {
-        const [listKey, jobId, activityId, type] = timeJob;
-        await timeEventCallback(jobId, activityId, type);
-        await sleepFor(0);          
-        this.processTimeHooks(timeEventCallback, listKey);
-      } else {
-        let sleep = XSleepFor(FIDELITY_SECONDS * 1000);
-        this.cleanupTimeout = sleep.timerId;
-        await sleep.promise;
-        this.processTimeHooks(timeEventCallback)
+  /**
+   * Should this engine instance play the role of 'scout' for the quorum.
+   */
+  async shouldScout() {
+    const wasScout = this.isScout;
+    const isScout = wasScout || (this.isScout = await this.store.reserveScoutRole('time'));
+    if (isScout) {
+      if (!wasScout) {
+        setTimeout(() => {
+          this.isScout = false;
+        }, SCOUT_INTERVAL_SECONDS * 1_000);
+      }
+      return true;
+    }
+    return false;
+  }
+
+  async processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt') => Promise<void>, listKey?: string): Promise<void> {
+    if (await this.shouldScout()) {
+      try {
+        const timeJob = await this.store.getNextTimeJob(listKey);
+        if (Array.isArray(timeJob)) {
+          //a queue had a job; try again immediately
+          const [listKey, jobId, gId, activityId, type] = timeJob;
+          await timeEventCallback(jobId, gId, activityId, type);
+          await sleepFor(0);
+          this.processTimeHooks(timeEventCallback, listKey);
+        } else if (timeJob) {
+          //a queue was just emptied; try again immediately
+          await sleepFor(0);
+          this.processTimeHooks(timeEventCallback);
+        } else {
+          //all queues are empty; sleep before checking
+          let sleep = XSleepFor(FIDELITY_SECONDS * 1000);
+          this.cleanupTimeout = sleep.timerId;
+          await sleep.promise;
+          this.processTimeHooks(timeEventCallback);
+        }
+      } catch (err) {
+        //todo: retry connect to redis
+        this.logger.error('task-process-timehooks-error', err);
       }
-    } catch (err) {
-      //todo: retry connect to redis
-      this.logger.error('task-process-timehooks-error', err);
+    } else {
+      //didn't get the scout role; try again in 'one-ish' minutes
+      let sleep = XSleepFor(SCOUT_INTERVAL_SECONDS * 1_000 * 2 * Math.random());
+      this.cleanupTimeout = sleep.timerId;
+      await sleep.promise;
+      this.processTimeHooks(timeEventCallback);
     }
   }
 
@@ -81,6 +115,71 @@ class TaskService {
       this.cleanupTimeout = undefined;
     }
   }
+
+  async getHookRule(topic: string): Promise<HookRule | undefined> {
+    const rules = await this.store.getHookRules();
+    return rules?.[topic]?.[0] as HookRule;
+  }
+
+  async registerWebHook(topic: string, context: JobState, dad: string, multi?: RedisMulti): Promise<string> {
+    const hookRule = await this.getHookRule(topic);
+    if (hookRule) {
+      const mapExpression = hookRule.conditions.match[0].expected;
+      const resolved = Pipe.resolve(mapExpression, context);
+      const jobId = context.metadata.jid;
+      const gId = context.metadata.gid;
+      const activityId = hookRule.to;
+      const hook: HookSignal = {
+        topic,
+        resolved,
+        jobId: `${activityId}::${dad}::${gId}::${jobId}`,
+      }
+      await this.store.setHookSignal(hook, multi);
+      return jobId;
+    } else {
+      throw new Error('signaler.registerWebHook:error: hook rule not found');
+    }
+  }
+
+  async processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<[string, string, string, string] | undefined> {
+    const hookRule = await this.getHookRule(topic);
+    if (hookRule) {
+      //NOTE: both formats are supported by the mapping engine:
+      //      `$self.hook.data` OR `$hook.data`
+      const context = { $self: { hook: { data }}, $hook: { data }};
+      const mapExpression = hookRule.conditions.match[0].actual;
+      const resolved = Pipe.resolve(mapExpression, context);
+      const hookSignalId = await this.store.getHookSignal(topic, resolved);
+      if (!hookSignalId) {
+        //messages can be double-processed; not an issue; return undefined
+        //users can also provide a bogus topic; not an issue; return undefined
+        return undefined;
+      }
+      //`aid` is part of composit key, but the hook `topic` is its public interface;
+      // this means that a new version of the graph can be deployed and the
+      // topic can be re-mapped to a different activity id. Outside callers
+      // can adhere to the unchanged contract (calling the same topic),
+      // while the internal system can be updated in real time as necessary.
+      const [_aid, dad, gid, ...jid] = hookSignalId.split('::');
+      return [jid.join('::'), hookRule.to, dad, gid];
+    } else {
+      throw new Error('signal-not-found');
+    }
+  }
+
+  async deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number> {
+    const hookRule = await this.getHookRule(topic);
+    if (hookRule) {
+      //NOTE: both formats are supported by the mapping engine:
+      //      `$self.hook.data` OR `$hook.data`
+      const context = { $self: { hook: { data }}, $hook: { data }};
+      const mapExpression = hookRule.conditions.match[0].actual;
+      const resolved = Pipe.resolve(mapExpression, context);
+      return await this.store.deleteHookSignal(topic, resolved);
+    } else {
+      throw new Error('signaler.process:error: hook rule not found');
+    }
+  }
 }
 
 export { TaskService };

package/services/telemetry/index.ts

@@ -126,20 +126,20 @@ class TelemetryService {
           return result;
         }, {})
       };
-      this.span.setAttributes(namespacedAtts as StringScalarType);
+      this.span?.setAttributes(namespacedAtts as StringScalarType);
     }
   }
 
   setActivityAttributes(attributes: StringScalarType): void {
-    this.span.setAttributes(attributes);
+    this.span?.setAttributes(attributes);
   }
 
   setStreamAttributes(attributes: StringScalarType): void {
-    this.span.setAttributes(attributes);
+    this.span?.setAttributes(attributes);
   }
 
   setJobAttributes(attributes: StringScalarType): void {
-    this.jobSpan.setAttributes(attributes);
+    this.jobSpan?.setAttributes(attributes);
   }
 
   endJobSpan(): void {
@@ -216,11 +216,11 @@ class TelemetryService {
   }
 
   setActivityError(message: string) {
-    this.span.setStatus({ code: SpanStatusCode.ERROR, message });
+    this.span?.setStatus({ code: SpanStatusCode.ERROR, message });
   }
 
   setStreamError(message: string) {
-    this.span.setStatus({ code: SpanStatusCode.ERROR, message });
+    this.span?.setStatus({ code: SpanStatusCode.ERROR, message });
   }
 
   /**