@hotmeshio/hotmesh 0.0.55 → 0.0.57

package/services/activities/cycle.ts

@@ -1,115 +0,0 @@
-import {
-  GenerationalError,
-  GetStateError,
-  InactiveJobError } from '../../modules/errors';
-import { guid } from '../../modules/utils';
-import { CollatorService } from '../collator';
-import { EngineService } from '../engine';
-import { Activity } from './activity';
-import {
-  ActivityData,
-  ActivityMetadata,
-  ActivityType,
-  CycleActivity } from '../../types/activity';
-import { JobState } from '../../types/job';
-import { MultiResponseFlags, RedisMulti } from '../../types/redis';
-import { StreamData } from '../../types/stream';
-import { TelemetryService } from '../telemetry';
-
-class Cycle extends Activity {
-  config: CycleActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService,
-    context?: JobState) {
-      super(config, data, metadata, hook, engine, context);
-  }
-
-
-  //********  LEG 1 ENTRY  ********//
-  async process(): Promise<string> {
-    this.logger.debug('cycle-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    let telemetry: TelemetryService;
-    try {
-      await this.verifyEntry();
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-      this.mapInputData();
-
-      //set state/status
-      let multi = this.store.getMulti();
-      await this.setState(multi);
-      await this.setStatus(0, multi); //leg 1 never changes job status
-      const multiResponse = await multi.exec() as MultiResponseFlags;
-      telemetry.mapActivityAttributes();
-      const jobStatus = this.resolveStatus(multiResponse);
-
-      //cycle the target ancestor
-      multi = this.store.getMulti();
-      const messageId = await this.cycleAncestorActivity(multi);
-      telemetry.setActivityAttributes({
-        'app.activity.mid': messageId,
-        'app.job.jss': jobStatus
-      });
-
-      //exit early (`Cycle` activities only execute Leg 1)
-      await CollatorService.notarizeEarlyExit(this, multi);
-      await multi.exec() as MultiResponseFlags;
-
-      return this.context.metadata.aid;
-    } catch (error) {
-      if (error instanceof InactiveJobError) {
-        this.logger.error('cycle-inactive-job-error', { ...error });
-        return;
-      } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { ...error });
-        return;
-      } else if (error instanceof GetStateError) {
-        this.logger.error('cycle-get-state-error', { ...error });
-        return;
-      } else {
-        this.logger.error('cycle-process-error', { ...error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry?.endActivitySpan();
-      this.logger.debug('cycle-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    }
-  }
-
-  /**
-   * Trigger the target ancestor to execute in a cycle,
-   * without violating the constraints of the DAG. Immutable
-   * `individual activity state` will execute in a new dimensional
-   * thread while `shared job state` can change. This
-   * pattern allows for retries without violating the DAG.
-   */
-  async cycleAncestorActivity(multi: RedisMulti): Promise<string> {
-    //Cycle activity L1 is a standin for the target ancestor L1.
-    //Input data mapping (mapInputData) allows for the 
-    //next dimensonal thread to execute with different
-    //input data than the current dimensional thread
-    this.mapInputData();
-    const streamData: StreamData = {
-      metadata: {
-        guid: guid(),
-        jid: this.context.metadata.jid,
-        gid: this.context.metadata.gid,
-        dad: CollatorService.resolveReentryDimension(this),
-        aid: this.config.ancestor,
-        spn: this.context['$self'].output.metadata?.l1s,
-        trc: this.context.metadata.trc,
-      },
-      data: this.context.data
-    };
-    return (await this.engine.router?.publishMessage(null, streamData, multi)) as string;
-  }
-}
-
-export { Cycle };

package/services/activities/hook.ts

@@ -1,197 +0,0 @@
-import {
-  GenerationalError,
-  GetStateError,
-  InactiveJobError } from '../../modules/errors';
-import { Activity } from './activity';
-import { CollatorService } from '../collator';
-import { EngineService } from '../engine';
-import { Pipe } from '../pipe';
-import { TaskService } from '../task';
-import { TelemetryService } from '../telemetry';
-import { 
-  ActivityData,
-  ActivityMetadata,
-  ActivityType,
-  HookActivity } from '../../types/activity';
-import { HookRule } from '../../types/hook';
-import { JobState, JobStatus } from '../../types/job';
-import {
-  MultiResponseFlags,
-  RedisMulti } from '../../types/redis';
-import { StringScalarType } from '../../types/serializer';
-import { StreamCode, StreamStatus } from '../../types/stream';
-
-/**
- * Supports `signal hook`, `time hook`, and `cycle hook` patterns
- */
-class Hook extends Activity {
-  config: HookActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService,
-    context?: JobState) {
-      super(config, data, metadata, hook, engine, context);
-  }
-
-  //********  INITIAL ENTRY POINT (A)  ********//
-  async process(): Promise<string> {
-    this.logger.debug('hook-process', {
-      jid: this.context.metadata.jid,
-      gid: this.context.metadata.gid,
-      aid: this.metadata.aid,
-    });
-    let telemetry: TelemetryService;
-
-    try {
-      await this.verifyEntry();
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-
-      if (this.doesHook()) {
-        //sleep and wait to awaken upon a signal
-        await this.doHook(telemetry);
-      } else {
-        //end the activity and transition to its children
-        await this.doPassThrough(telemetry);
-      }
-
-      return this.context.metadata.aid;
-    } catch (error) {
-      if (error instanceof InactiveJobError) {
-        this.logger.error('hook-inactive-job-error', { ...error });
-        return;
-      } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { ...error });
-        return;
-      } else if (error instanceof GetStateError) {
-        this.logger.error('hook-get-state-error', { ...error });
-        return;
-      } else {
-        this.logger.error('hook-process-error', { ...error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry?.endActivitySpan();
-      this.logger.debug('hook-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    }
-  }
-
-  /**
-   * does this activity use a time-hook or web-hook
-   */
-  doesHook(): boolean {
-    if (this.config.sleep) {
-      const duration = Pipe.resolve(
-        this.config.sleep,
-        this.context,
-      );
-      return !isNaN(duration) && Number(duration) > 0
-    }
-    return !!this.config.hook?.topic;
-  }
-
-  async doHook(telemetry: TelemetryService) {
-    const multi = this.store.getMulti();
-    await this.registerHook(multi);
-    this.mapOutputData();
-    this.mapJobData();
-    await this.setState(multi);
-    await CollatorService.authorizeReentry(this, multi);
-
-    await this.setStatus(0, multi);
-    await multi.exec();
-    telemetry.mapActivityAttributes();
-  }
-
-  async doPassThrough(telemetry: TelemetryService) {
-    const multi = this.store.getMulti();
-    let multiResponse: MultiResponseFlags;
-    
-    this.adjacencyList = await this.filterAdjacent();
-    this.mapOutputData();
-    this.mapJobData();
-    await this.setState(multi);
-    await CollatorService.notarizeEarlyCompletion(this, multi);
-
-    await this.setStatus(this.adjacencyList.length - 1, multi);
-    multiResponse = await multi.exec() as MultiResponseFlags;
-    telemetry.mapActivityAttributes();
-    const jobStatus = this.resolveStatus(multiResponse);
-    const attrs: StringScalarType = { 'app.job.jss': jobStatus };
-    const messageIds = await this.transition(this.adjacencyList, jobStatus);
-    if (messageIds.length) {
-      attrs['app.activity.mids'] = messageIds.join(',')
-    }
-    telemetry.setActivityAttributes(attrs);
-  }
-
-  async getHookRule(topic: string): Promise<HookRule | undefined> {
-    const rules = await this.store.getHookRules();
-    return rules?.[topic]?.[0] as HookRule;
-  }
-
-  async registerHook(multi?: RedisMulti): Promise<string | void> {
-    if (this.config.hook?.topic) {
-      return await this.engine.taskService.registerWebHook(
-        this.config.hook.topic,
-        this.context,
-        this.resolveDad(),
-        multi
-      );
-    } else if (this.config.sleep) {
-      const duration = Pipe.resolve(
-        this.config.sleep,
-        this.context,
-      );
-      await this.engine.taskService.registerTimeHook(
-        this.context.metadata.jid,
-        this.context.metadata.gid,
-        `${this.metadata.aid}${this.metadata.dad || ''}`,
-        'sleep',
-        duration,
-        this.metadata.dad || '',
-      );
-      return this.context.metadata.jid;
-    }
-  }
-
-  //********  SIGNAL RE-ENTRY POINT  ********//
-  async processWebHookEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<JobStatus | void> {
-    this.logger.debug('hook-process-web-hook-event', {
-      topic: this.config.hook.topic,
-      aid: this.metadata.aid,
-      status,
-      code,
-    });
-    const taskService = new TaskService(this.store, this.logger);
-    const data = { ...this.data };
-    const signal = await taskService.processWebHookSignal(this.config.hook.topic, data);
-    if (signal) {
-      const [jobId, _aid, dad, gId] = signal;
-      this.context.metadata.jid = jobId;
-      this.context.metadata.gid = gId;
-      this.context.metadata.dad = dad;
-      await this.processEvent(status, code, 'hook');
-      if (code === 200) {
-        await taskService.deleteWebHookSignal(this.config.hook.topic, data);
-      } //else => 202/keep alive
-    } //else => already resolved
-  }
-
-  async processTimeHookEvent(jobId: string): Promise<JobStatus | void> {
-    this.logger.debug('hook-process-time-hook-event', {
-      jid: jobId,
-      gid: this.context.metadata.gid,
-      aid: this.metadata.aid
-    });
-    await this.processEvent(StreamStatus.SUCCESS, 200, 'hook');
-  }
-}
-
-export { Hook };

package/services/activities/index.ts

@@ -1,19 +0,0 @@
-import { Activity } from './activity';
-import { Await } from './await';
-import { Cycle } from './cycle';
-import { Hook } from './hook';
-import { Interrupt } from './interrupt';
-import { Signal } from './signal';
-import { Trigger } from './trigger';
-import { Worker } from './worker';
-
-export default { 
-  activity: Activity,
-  await: Await,
-  cycle: Cycle,
-  hook: Hook,
-  interrupt: Interrupt,
-  signal: Signal,
-  trigger: Trigger,
-  worker: Worker,
-};

package/services/activities/interrupt.ts

@@ -1,172 +0,0 @@
-import {
-  GenerationalError,
-  GetStateError,
-  InactiveJobError } from '../../modules/errors';
-import { CollatorService } from '../collator';
-import { Activity } from './activity';
-import { EngineService } from '../engine';
-import { Pipe } from '../pipe';
-import { TelemetryService } from '../telemetry';
-import {
-  ActivityData,
-  ActivityMetadata,
-  ActivityType,
-  InterruptActivity } from '../../types/activity';
-import { JobInterruptOptions, JobState } from '../../types/job';
-import { MultiResponseFlags } from '../../types/redis';
-
-class Interrupt extends Activity {
-  config: InterruptActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService,
-    context?: JobState
-    ) {
-      super(config, data, metadata, hook, engine, context);
-  }
-
-
-  //********  LEG 1 ENTRY  ********//
-  async process(): Promise<string> {
-    this.logger.debug('interrupt-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    let telemetry: TelemetryService;
-    try {
-      await this.verifyEntry();
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-  
-      if (this.isInterruptingSelf()) {
-        await this.interruptSelf(telemetry);
-      } else {
-        await this.interruptAnother(telemetry);
-      }
-    } catch (error) {
-      if (error instanceof InactiveJobError) {
-        this.logger.error('interrupt-inactive-job-error', { ...error });
-        return;
-      } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { ...error });
-        return;
-      } else if (error instanceof GetStateError) {
-        this.logger.error('interrupt-get-state-error', { ...error });
-        return;
-      } else {
-        this.logger.error('interrupt-process-error', { ...error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry?.endActivitySpan();
-      this.logger.debug('interrupt-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    }
-  }
-
-  async interruptSelf(telemetry: TelemetryService): Promise<string> {
-    // Apply final updates to THIS job's state
-    if (this.config.job?.maps) {
-      this.mapJobData();
-      await this.setState();
-    }
-  
-    // Interrupt THIS job
-    const messageId = await this.interrupt();
-  
-    // Notarize completion and log
-    telemetry.mapActivityAttributes();
-    const multi = this.store.getMulti();
-    await CollatorService.notarizeEarlyCompletion(this, multi);
-    await this.setStatus(-1, multi);
-    const multiResponse = await multi.exec() as MultiResponseFlags;
-    const jobStatus = this.resolveStatus(multiResponse);
-    telemetry.setActivityAttributes({
-      'app.activity.mid': messageId,
-      'app.job.jss': jobStatus
-    });
-  
-    return this.context.metadata.aid;
-  }
-
-  async interruptAnother(telemetry: TelemetryService): Promise<string> {
-    // Interrupt ANOTHER job
-    const messageId = await this.interrupt();
-    const attrs = { 'app.activity.mid': messageId };
-  
-    // Apply updates to THIS job's state
-    telemetry.mapActivityAttributes();
-    this.adjacencyList = await this.filterAdjacent();
-    if (this.config.job?.maps || this.config.output?.maps) {
-      this.mapOutputData();
-      this.mapJobData();
-      const multi = this.store.getMulti();
-      await this.setState(multi);
-    }
-  
-    // Notarize completion
-    const multi = this.store.getMulti();
-    await CollatorService.notarizeEarlyCompletion(this, multi);
-    await this.setStatus(this.adjacencyList.length - 1, multi);
-    const multiResponse = await multi.exec() as MultiResponseFlags;
-    const jobStatus = this.resolveStatus(multiResponse);
-    attrs['app.job.jss'] = jobStatus;
-  
-    // Transition next generation and log
-    const messageIds = await this.transition(this.adjacencyList, jobStatus);
-    if (messageIds.length) {
-      attrs['app.activity.mids'] = messageIds.join(',');
-    }
-    telemetry.setActivityAttributes(attrs);
-  
-    return this.context.metadata.aid;
-  }
-
-  isInterruptingSelf(): boolean {
-    if (!this.config.target) {
-      return true;
-    }
-    const resolvedJob = Pipe.resolve(this.config.target, this.context);
-    return resolvedJob == this.context.metadata.jid;
-  }
-
-  resolveInterruptOptions(): JobInterruptOptions {
-    return {
-      reason: this.config.reason !== undefined 
-        ? Pipe.resolve(this.config.reason, this.context) 
-        : undefined,
-      throw: this.config.throw !== undefined 
-        ? Pipe.resolve(this.config.throw, this.context) 
-        : undefined,
-      descend: this.config.descend !== undefined 
-        ? Pipe.resolve(this.config.descend, this.context) 
-        : undefined,
-      code: this.config.code !== undefined 
-        ? Pipe.resolve(this.config.code, this.context) 
-        : undefined,
-      expire: this.config.expire !== undefined
-        ? Pipe.resolve(this.config.expire, this.context)
-        : undefined,
-      stack: this.config.stack !== undefined
-        ? Pipe.resolve(this.config.stack, this.context)
-        : undefined,
-    };
-  }
-
-  async interrupt(): Promise<string> {
-    const options = this.resolveInterruptOptions();
-    return await this.engine.interrupt(
-      this.config.topic !== undefined 
-        ? Pipe.resolve(this.config.topic, this.context) 
-        : this.context.metadata.tpc,
-      this.config.target !== undefined 
-        ? Pipe.resolve(this.config.target, this.context) 
-        : this.context.metadata.jid,
-      options as JobInterruptOptions,
-    );
-  }
-}
-
-export { Interrupt };

package/services/activities/signal.ts

@@ -1,148 +0,0 @@
-import {
-  GenerationalError,
-  GetStateError,
-  InactiveJobError } from '../../modules/errors';
-import { Activity } from './activity';
-import { CollatorService } from '../collator';
-import { EngineService } from '../engine';
-import { MapperService } from '../mapper';
-import { Pipe } from '../pipe';
-import { TelemetryService } from '../telemetry';
-import {
-  ActivityData,
-  ActivityMetadata,
-  ActivityType,
-  SignalActivity } from '../../types/activity';
-import { JobState } from '../../types/job';
-import { MultiResponseFlags } from '../../types/redis';
-import { StringScalarType } from '../../types/serializer';
-import { JobStatsInput } from '../../types/stats';
-
-class Signal extends Activity {
-  config: SignalActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService,
-    context?: JobState) {
-      super(config, data, metadata, hook, engine, context);
-  }
-
-
-  //********  LEG 1 ENTRY  ********//
-  async process(): Promise<string> {
-    this.logger.debug('signal-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    let telemetry: TelemetryService;
-    try {
-      await this.verifyEntry();
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-
-      //save state and notarize early completion (signals only run leg1)
-      const multi = this.store.getMulti();
-      this.adjacencyList = await this.filterAdjacent();
-      this.mapOutputData();
-      this.mapJobData();
-      await this.setState(multi);
-      await CollatorService.notarizeEarlyCompletion(this, multi);
-      await this.setStatus(this.adjacencyList.length - 1, multi);
-      const multiResponse = await multi.exec() as MultiResponseFlags;
-
-      //todo: this should execute BEFORE the status is decremented
-      if (this.config.subtype === 'all') {
-        await this.hookAll();
-      } else {
-        await this.hookOne();
-      }
-
-      //transition to adjacent activities
-      const jobStatus = this.resolveStatus(multiResponse);
-      const attrs: StringScalarType = { 'app.job.jss': jobStatus };
-      const messageIds = await this.transition(this.adjacencyList, jobStatus);
-      if (messageIds.length) {
-        attrs['app.activity.mids'] = messageIds.join(',')
-      }
-      telemetry.mapActivityAttributes();
-      telemetry.setActivityAttributes(attrs);
-
-      return this.context.metadata.aid;
-    } catch (error) {
-      if (error instanceof InactiveJobError) {
-        this.logger.error('signal-inactive-job-error', { ...error });
-        return;
-      } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { ...error });
-        return;
-      } else if (error instanceof GetStateError) {
-        this.logger.error('signal-get-state-error', { ...error });
-        return;
-      } else {
-        this.logger.error('signal-process-error', { ...error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry?.endActivitySpan();
-      this.logger.debug('signal-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    }
-  }
-
-  mapSignalData(): Record<string, any> {
-    if(this.config.signal?.maps) {
-      const mapper = new MapperService(this.config.signal.maps, this.context);
-      return mapper.mapRules();
-    }
-  }
-
-  mapResolverData(): Record<string, any> {
-    if(this.config.resolver?.maps) {
-      const mapper = new MapperService(this.config.resolver.maps, this.context);
-      return mapper.mapRules();
-    }
-  }
-
-  /**
-   * The signal activity will hook one
-   */
-  async hookOne(): Promise<string> {
-    const topic = Pipe.resolve(this.config.topic, this.context);
-    const signalInputData = this.mapSignalData();
-    const status = Pipe.resolve(this.config.status, this.context);
-    const code = Pipe.resolve(this.config.code, this.context);
-    return await this.engine.hook(topic, signalInputData, status, code);
-  }
-
-  /**
-   * The signal activity will hook all paused jobs that share the same job key.
-   */
-  async hookAll(): Promise<string[]> {
-    //prep 1) generate `input signal data` (essentially the webhook payload)
-    const signalInputData = this.mapSignalData();
-
-    //prep 2) generate data that resolves the job key (per the YAML config)
-    const keyResolverData = this.mapResolverData() as JobStatsInput;
-    if (this.config.scrub) {
-      //self-clean the indexes upon use if configured
-      keyResolverData.scrub = true;
-    }
-
-    //prep 3) jobKeys can contain multiple indexes (per the YAML config)
-    const key_name = Pipe.resolve(this.config.key_name, this.context);
-    const key_value = Pipe.resolve(this.config.key_value, this.context);
-    const indexQueryFacets = [`${key_name}:${key_value}`];
-
-    //execute: `hookAll` will now resume all paused jobs that share the same job key
-    return await this.engine.hookAll(
-      this.config.topic,
-      signalInputData,
-      keyResolverData,
-      indexQueryFacets
-    );
-  }
-}
-
-export { Signal };