@hotmeshio/hotmesh 0.0.55 → 0.0.57

package/services/activities/trigger.ts

@@ -1,295 +0,0 @@
-import { DuplicateJobError } from '../../modules/errors';
-import {
-  formatISODate,
-  getTimeSeries,
-  guid } from '../../modules/utils';
-import { Activity } from './activity';
-import { CollatorService } from '../collator';
-import { EngineService } from '../engine';
-import { Pipe } from '../pipe';
-import { ReporterService } from '../reporter';
-import { MDATA_SYMBOLS } from '../serializer';
-import { TelemetryService } from '../telemetry';
-import {
-  ActivityData,
-  ActivityMetadata,
-  ActivityType,
-  TriggerActivity } from '../../types/activity';
-import { JobState, ExtensionType } from '../../types/job';
-import { RedisMulti } from '../../types/redis';
-import { StringScalarType } from '../../types/serializer';
-import { WorkListTaskType } from '../../types/task';
-
-class Trigger extends Activity {
-  config: TriggerActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService,
-    context?: JobState) {
-      super(config, data, metadata, hook, engine, context);
-  }
-
-  async process(options?: ExtensionType): Promise<string> {
-    this.logger.debug('trigger-process', { subscribes: this.config.subscribes });
-    let telemetry: TelemetryService;
-    try {
-      this.setLeg(2);
-      await this.getState();
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startJobSpan();
-      telemetry.startActivitySpan(this.leg);
-      this.mapJobData();
-      await this.setStateNX();
-      this.adjacencyList = await this.filterAdjacent();
-      await this.setStatus(this.adjacencyList.length);
-
-      this.bindSearchData(options);
-      this.bindMarkerData(options);
-
-      const multi = this.store.getMulti();
-      await this.setState(multi);
-      await this.setStats(multi);
-      await this.registerJobDependency(multi);
-      await multi.exec();
-
-      //if the parent (spawner) chose not to await,
-      // emit the job_id as the data payload { job_id }
-      this.execAdjacentParent();
-
-      telemetry.mapActivityAttributes();
-      const jobStatus = Number(this.context.metadata.js);
-      telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
-      const attrs: StringScalarType = { 'app.job.jss': jobStatus };
-      const messageIds = await this.transition(this.adjacencyList, jobStatus);
-      if (messageIds.length) {
-        attrs['app.activity.mids'] = messageIds.join(',')
-      }
-      telemetry.setActivityAttributes(attrs);
-      return this.context.metadata.jid;
-    } catch (error) {
-      if (error instanceof DuplicateJobError) {
-        this.logger.error('duplicate-job-error', { job_id: error.jobId });
-      } else {
-        this.logger.error('trigger-process-error', { ...error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry?.endJobSpan();
-      telemetry?.endActivitySpan();
-      this.logger.debug('trigger-process-end', { subscribes: this.config.subscribes, jid: this.context.metadata.jid, gid: this.context.metadata.gid });
-    }
-  }
-
-  safeKey(key:string): string {
-    return `_${key}`;
-  }
-
-  bindSearchData(options?: ExtensionType): void {
-    if (options?.search) {
-      Object.keys(options.search).forEach((key) => {
-        this.context.data[this.safeKey(key)] = options.search[key].toString();
-      });
-    }
-  }
-
-  bindMarkerData(options?: ExtensionType): void {
-    if (options?.marker) {
-      Object.keys(options.marker).forEach((key) => {
-        if (key.startsWith('-')) {
-          this.context.data[key] = options.marker[key].toString();
-        }
-      });
-    }
-  }
-
-  async setStatus(amount: number): Promise<void> {
-    this.context.metadata.js = amount;
-  }
-
-  async execAdjacentParent() {
-    if (this.context.metadata.px) {
-      const timestamp = formatISODate(new Date());
-      const jobStartedConfirmationMessage = {
-        metadata: this.context.metadata,
-        data: {
-          job_id: this.context.metadata.jid,
-          jc: timestamp,
-          ju: timestamp,
-        }
-      };
-      await this.engine.execAdjacentParent(this.context, jobStartedConfirmationMessage);
-    }
-  }
-
-  createInputContext(): Partial<JobState> {
-    const input = { 
-      [this.metadata.aid]: {
-        input: { data: this.data }
-      },
-      '$self': {
-        input: { data: this.data },
-        output: { data: this.data }
-      },
-    } as Partial<JobState>;
-    return input
-  }
-
-  async getState(): Promise<void> {
-    const inputContext = this.createInputContext();
-    const jobId = this.resolveJobId(inputContext);
-    const jobKey = this.resolveJobKey(inputContext);
-
-    const utc = formatISODate(new Date());
-    const { id, version } = await this.engine.getVID();
-    this.initDimensionalAddress(CollatorService.getDimensionalSeed());
-    const activityMetadata = {
-      ...this.metadata,
-      jid: jobId,
-      key: jobKey,
-      as: CollatorService.getTriggerSeed(),
-    };
-    this.context = {
-      metadata: {
-        ...this.metadata,
-        gid: guid(),
-        ngn: this.context.metadata.ngn,
-        pj: this.context.metadata.pj,
-        pg: this.context.metadata.pg,
-        pd: this.context.metadata.pd,
-        pa: this.context.metadata.pa,
-        px: this.context.metadata.px,
-        app: id,
-        vrs: version,
-        tpc: this.config.subscribes,
-        trc: this.context.metadata.trc,
-        spn: this.context.metadata.spn,
-        jid: jobId,
-        dad: CollatorService.getDimensionalSeed(), //top-level job implicitly uses `,0`
-        key: jobKey,
-        jc: utc,
-        ju: utc,
-        ts: getTimeSeries(this.resolveGranularity()),
-        js: 0,
-      },
-      data: {},
-      [this.metadata.aid]: { 
-        input: { 
-          data: this.data,
-          metadata: activityMetadata
-        },
-        output: { 
-          data: this.data,
-          metadata: activityMetadata
-        },
-        settings: { data: {} },
-        errors: { data: {} },
-       },
-    };
-    this.context['$self'] = this.context[this.metadata.aid];
-    this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
-  }
-
-  bindJobMetadataPaths(): string[] {
-    return MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
-  }
-
-  bindActivityMetadataPaths(): string[] {
-    return MDATA_SYMBOLS.ACTIVITY.KEYS.map((key) => `output/metadata/${key}`);
-  }
-
-  resolveGranularity(): string {
-    return this.config.stats?.granularity || ReporterService.DEFAULT_GRANULARITY;
-  }
-
-  getJobStatus(): number {
-    return this.context.metadata.js;
-  }
-
-  resolveJobId(context: Partial<JobState>): string {
-    const jobId = this.config.stats?.id;
-    return jobId ? Pipe.resolve(jobId, context) : guid();
-  }
-
-  resolveJobKey(context: Partial<JobState>): string {
-    const jobKey = this.config.stats?.key;
-    return jobKey ? Pipe.resolve(jobKey, context) : '';
-  }
-
-  async setStateNX(): Promise<void> {
-    const jobId = this.context.metadata.jid;
-    if (!await this.store.setStateNX(jobId, this.engine.appId)) {
-      throw new DuplicateJobError(jobId);
-    }
-  }
-
-  /**
-   * Registers this job as a dependent of the parent job; when the
-   * parent job is interrupted, this job will be interrupted
-   */
-  async registerJobDependency(multi?: RedisMulti): Promise<void> {
-    const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
-    let resolvedDepKey = depKey ? Pipe.resolve(depKey, this.context) : '';
-    const adjKey = this.config.stats?.adjacent;
-    let resolvedAdjKey = depKey ? Pipe.resolve(adjKey, this.context) : '';
-    if (!resolvedDepKey) {
-      resolvedDepKey = this.context.metadata.pj;
-    }
-    if (resolvedDepKey) {
-      const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
-      let type: WorkListTaskType;
-      if (isParentOrigin) {
-        if (this.context.metadata.px) {
-          type = 'child'
-        } else {
-          type = 'expire-child'
-        }
-      } else {
-        type = 'expire';
-      }
-      await this.store.registerJobDependency(
-        type,
-        resolvedDepKey,
-        this.context.metadata.tpc,
-        this.context.metadata.jid,
-        this.context.metadata.gid,
-        this.context.metadata.pd,
-        multi,
-      );
-    }
-    if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
-      await this.store.registerJobDependency(
-        'child',
-        resolvedAdjKey,
-        this.context.metadata.tpc,
-        this.context.metadata.jid,
-        this.context.metadata.gid,
-        this.context.metadata.pd,
-        multi,
-      );
-    }
-  }
-
-  async setStats(multi?: RedisMulti): Promise<void> {
-    const md = this.context.metadata;
-    if (md.key && this.config.stats?.measures) {
-      const config = await this.engine.getVID();
-      const reporter = new ReporterService(config, this.store, this.logger);
-      await this.store.setStats(
-        md.key,
-        md.jid,
-        md.ts,
-        reporter.resolveTriggerStatistics(this.config, this.context),
-        config,
-        multi
-      );
-    }
-  }
-}
-
-export { Trigger };

package/services/activities/worker.ts

@@ -1,107 +0,0 @@
-import {
-  GenerationalError,
-  GetStateError,
-  InactiveJobError } from '../../modules/errors';
-import { guid } from '../../modules/utils';
-import { Activity } from './activity';
-import { CollatorService } from '../collator';
-import { EngineService } from '../engine';
-import { Pipe } from '../pipe';
-import { TelemetryService } from '../telemetry';
-import {
-  ActivityData,
-  ActivityMetadata,
-  ActivityType,
-  WorkerActivity } from '../../types/activity';
-import { JobState } from '../../types/job';
-import { MultiResponseFlags, RedisMulti } from '../../types/redis';
-import { StreamData} from '../../types/stream';
-
-class Worker extends Activity {
-  config: WorkerActivity;
-
-  constructor(
-    config: ActivityType,
-    data: ActivityData,
-    metadata: ActivityMetadata,
-    hook: ActivityData | null,
-    engine: EngineService,
-    context?: JobState) {
-      super(config, data, metadata, hook, engine, context);
-  }
-
-  //********  INITIAL ENTRY POINT (A)  ********//
-  async process(): Promise<string> {
-    this.logger.debug('worker-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    let telemetry: TelemetryService;
-    try {
-      await this.verifyEntry();
-
-      telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
-      telemetry.startActivitySpan(this.leg);
-      this.mapInputData();
-
-      //save state and authorize reentry
-      const multi = this.store.getMulti();
-      //todo: await this.registerTimeout();
-      const messageId = await this.execActivity(multi);
-      await CollatorService.authorizeReentry(this, multi);
-      await this.setState(multi);
-      await this.setStatus(0, multi);
-      const multiResponse = await multi.exec() as MultiResponseFlags;
-
-      //telemetry
-      telemetry.mapActivityAttributes();
-      const jobStatus = this.resolveStatus(multiResponse);
-      telemetry.setActivityAttributes({
-        'app.activity.mid': messageId,
-        'app.job.jss': jobStatus
-      });
-
-      return this.context.metadata.aid;
-    } catch (error) {
-      if (error instanceof InactiveJobError) {
-        this.logger.error('await-inactive-job-error', { ...error });
-        return;
-      } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { ...error });
-        return;
-      } else if (error instanceof GetStateError) {
-        this.logger.error('worker-get-state-error', { ...error });
-        return;
-      } else {
-        this.logger.error('worker-process-error', { ...error });
-      }
-      telemetry.setActivityError(error.message);
-      throw error;
-    } finally {
-      telemetry?.endActivitySpan();
-      this.logger.debug('worker-process-end', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
-    }
-  }
-
-  async execActivity(multi: RedisMulti): Promise<string> {
-    const topic = Pipe.resolve(this.config.subtype, this.context);
-    const streamData: StreamData = {
-      metadata: {
-        guid: guid(),
-        jid: this.context.metadata.jid,
-        gid: this.context.metadata.gid,
-        dad: this.metadata.dad,
-        aid: this.metadata.aid,
-        topic,
-        spn: this.context['$self'].output.metadata.l1s,
-        trc: this.context.metadata.trc,
-      },
-      data: this.context.data
-    };
-    if (this.config.retry) {
-      streamData.policies = {
-        retry: this.config.retry
-      };
-    }
-    return (await this.engine.router?.publishMessage(topic, streamData, multi)) as string;
-  }
-}
-
-export { Worker };

package/services/collator/README.md

@@ -1,102 +0,0 @@
-# Activity State Tracking
-
-This document explains how HotMesh guarantees activity state management using a 15-digit integer. This mechanism is crucial for monitoring and controlling the activity's lifecycle across two legs (Leg1 and Leg2) and ensures that duplicate activities are never processed. The process further ensures that catastrophic, in-process failures will be automatically resolved with respect to idempotency, makig it possible to build 100% fault-tolerant workflows.
-
-## Overview
-
-The 15-digit integer serves as a semaphore, starting with the value `999000000000000`. Each digit has a specific purpose:
-
-- The first three digits track the lifecycle status for Legs 1 and 2.
-- The last 12 digits represent 1 million possible dimensional threads that can be spawned by this activity.
-
-### Digit Details
-
-    999000000000000
-    ^-------------- Leg1 Entry Status
-     ^------------- Leg1 Exit Status
-       ^^^^^^------ Leg2 Dimensional Thread Entry Count
-             ^^^^^^ Leg2 Dimensional Thread Exit Count
-      ^------------ Leg2 Exit Status
-
->*Dimensional Threads* isolate and track those activities in the workflow that run in a *cycle*. They ensure that no naming collisions occur, even if the same activity is run multiple times.
-
-## Leg1 Lifecycle
-
-### Initialization
-
-The parent activity initializes the integer value (`999000000000000`) and saves it to Redis upon saving its own state. The parent then adds the child activity to the proper stream channel for eventual processing.
-
->Streams are used when executing an activity (such as transitioning to a child activity) as they guarantee that the child activity will be fully created and initialized before the request is marked for deletion. Even if the system has a catastrophic failure, the chain of custody can be guaranteed through the use of streams when the system comes online.
-
-### Beginning of Leg1
-
-When Leg1 begins (when an activity is dequeued from its stream channel), the integer is decremented by 100 trillion:
-
-    HINCRBY -100000000000000
-
-Result:
-
-    899000000000000
-
-### Conclusion of Leg1
-
-At the conclusion of Leg1, the integer is decremented by 10 trillion:
-
-    HINCRBY -10000000000000
-
-Result:
-
-    889000000000000
-
-### Error Handling
-
-- If the value upon entering is `799############` (after decrementing the integer), Leg1 began but crashed before completion. The activity will perform the necessary cleanup and re-run the activity.
-- If the value upon entering is `789############` (after decrementing the integer), Leg1 completed successfully and the activity should end. It is likely that the system crashed last time before acking and deleting. Verify transitions succeeded and resolve as necessary.
-
-## Leg2 Lifecycle
-
-Leg2 supports multiple inputs and repeated responses. A worker can respond with a 'pending' status, allowing multiple inputs.
-
-### Beginning of Leg2
-
-On the first Leg2 input, the integer is incremented by 1 million:
-
-    HINCRBY 1000000
-
-Result:
-
-    889000001000000
-
-### Pending Status
-
-If the call status is 'pending', the digit is updated to reflect the successful completion of Leg2 but the third digit from the left will be untouched as the activity is pending and Leg2 should not end quite yet:
-
-    HINCRBY 1
-
-Result:
-
-    889000001000001
-
-### Success Status
-
-If the next message call to Leg2 entry is 'success', the process will begin as before, and the integer will be incremented by 1 million:
-
-    HINCRBY 1000000
-
-Result:
-
-    889000002000001
-
- But because this is not a 'pending' message, the integer should be updated differently, targeting both the dimensional thread counter as well as the third digit from the left that tracks Leg2 status (e.g., `1 - 1000000000 = -999999999`).
-
-    HINCRBY -999999999999
-
-Result:
-
-    888000002000002
-
->If an additional Leg2 call were to be received after this point, the result would be `888000003000002`. However, becuase the third digit is `8`, Leg2 access is not allowed and the system will conclude by decrementing the value it just set and returning immediately, so that the calling worker can mark and delete the stream entry.
-
-## Conclusion
-
-This 15-digit integer allows sophisticated tracking of complex activities across two legs, with error handling and support for cycles using multiple dimensional threads.