@hotmeshio/hotmesh 0.2.4 → 0.3.1

package/build/services/store/index.js

@@ -33,6 +33,7 @@ const cache_1 = require("./cache");
 class StoreService {
     constructor(redisClient) {
         this.commands = {
+            get: 'get',
             set: 'set',
             setnx: 'setnx',
             del: 'del',
@@ -355,27 +356,6 @@ class StoreService {
         return await this.redisClient[this.commands.hset](key, payload);
     }
     async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
-        const privateMulti = multi || this.getMulti();
-        const dependencyParams = {
-            appId: this.appId,
-            jobId: originJobId,
-        };
-        const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
-        const expireTask = [depType, topic, gId, pd, jobId].join(key_1.VALSEP);
-        privateMulti[this.commands.rpush](depKey, expireTask);
-        if (!multi) {
-            return await privateMulti.exec();
-        }
-    }
-    async registerSignalDependency(jobId, signalKey, dad, multi) {
-        const privateMulti = multi || this.getMulti();
-        const dependencyParams = { appId: this.appId, jobId };
-        const dependencyKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
-        const delistTask = ['delist', 'signal', jobId, dad, signalKey].join(key_1.VALSEP);
-        privateMulti[this.commands.rpush](dependencyKey, delistTask);
-        if (!multi) {
-            return await privateMulti.exec();
-        }
     }
     async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
         const params = {
@@ -561,7 +541,7 @@ class StoreService {
             appId: this.appId,
             jobId,
         });
-        return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
+        return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount.toString());
     }
     async setStateNX(jobId, appId, status) {
         const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
@@ -718,19 +698,16 @@ class StoreService {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
         const { topic, resolved, jobId } = hook;
         const signalKey = `${topic}:${resolved}`;
-        const payload = { [signalKey]: jobId };
-        await (multi || this.redisClient)[this.commands.hset](key, payload);
-        const [_aid, dad, _gid, jid] = jobId.split(key_1.VALSEP);
-        return await this.registerSignalDependency(jid, signalKey, dad, multi);
+        await this.setnxex(`${key}:${signalKey}`, jobId, Math.max(hook.expire, enums_1.HMSH_SIGNAL_EXPIRE));
     }
     async getHookSignal(topic, resolved) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
-        const response = await this.redisClient[this.commands.hget](key, `${topic}:${resolved}`);
+        const response = await this.redisClient[this.commands.get](`${key}:${topic}:${resolved}`);
         return response ? response.toString() : undefined;
     }
     async deleteHookSignal(topic, resolved) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
-        const response = await this.redisClient[this.commands.hdel](key, `${topic}:${resolved}`);
+        const response = await this.redisClient[this.commands.del](`${key}:${topic}:${resolved}`);
         return response ? Number(response) : undefined;
     }
     async addTaskQueues(keys) {
@@ -779,14 +756,6 @@ class StoreService {
             await (redisMulti || this.redisClient)[this.commands.expire](jobKey, inSeconds);
         }
     }
-    async registerDependenciesForCleanup(jobId, deletionTime, options) {
-        const depParams = { appId: this.appId, jobId };
-        const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
-        const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
-        const depKeyContext = `${key_1.TYPSEP}${context}${key_1.TYPSEP}${depKey}`;
-        const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
-        await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
-    }
     async getDependencies(jobId) {
         const depParams = { appId: this.appId, jobId };
         const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);

package/build/services/task/index.d.ts

@@ -20,7 +20,7 @@ declare class TaskService {
     processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
     cancelCleanup(): void;
     getHookRule(topic: string): Promise<HookRule | undefined>;
-    registerWebHook(topic: string, context: JobState, dad: string, multi?: RedisMulti): Promise<string>;
+    registerWebHook(topic: string, context: JobState, dad: string, expire: number, multi?: RedisMulti): Promise<string>;
     processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<[string, string, string, string] | undefined>;
     deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
 }

package/build/services/task/index.js

@@ -36,10 +36,6 @@ class TaskService {
     async registerJobForCleanup(jobId, inSeconds = enums_1.HMSH_EXPIRE_DURATION, options) {
         if (inSeconds > 0) {
             await this.store.expireJob(jobId, inSeconds);
-            const fromNow = Date.now() + inSeconds * 1000;
-            const fidelityMS = enums_1.HMSH_FIDELITY_SECONDS * 1000;
-            const timeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
-            await this.store.registerDependenciesForCleanup(jobId, timeSlot, options);
         }
     }
     async registerTimeHook(jobId, gId, activityId, type, inSeconds = enums_1.HMSH_FIDELITY_SECONDS, dad, multi) {
@@ -73,7 +69,7 @@ class TaskService {
                         const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, {
                             appId: this.store.appId,
                         });
-                        await this.store.redisClient[this.store.commands.hdel](key, target);
+                        await this.store.redisClient[this.store.commands.del](`${key}:${target}`);
                     }
                     else {
                         await timeEventCallback(target, gId, activityId, type);
@@ -120,7 +116,7 @@ class TaskService {
         const rules = await this.store.getHookRules();
         return rules?.[topic]?.[0];
     }
-    async registerWebHook(topic, context, dad, multi) {
+    async registerWebHook(topic, context, dad, expire, multi) {
         const hookRule = await this.getHookRule(topic);
         if (hookRule) {
             const mapExpression = hookRule.conditions.match[0].expected;
@@ -133,6 +129,7 @@ class TaskService {
                 topic,
                 resolved,
                 jobId: compositeJobKey,
+                expire,
             };
             await this.store.setHookSignal(hook, multi);
             return jobId;

package/build/types/activity.d.ts

@@ -17,6 +17,8 @@ interface BaseActivity {
     emit?: boolean;
     sleep?: number;
     expire?: number;
+    persistent?: boolean;
+    persist?: boolean;
     retry?: StreamRetryPolicy;
     cycle?: boolean;
     collationInt?: number;

package/build/types/error.d.ts

@@ -4,6 +4,7 @@ export type MeshFlowChildErrorType = {
     backoffCoefficient?: number;
     index: number;
     expire?: number;
+    persistent?: boolean;
     signalIn?: boolean;
     maximumAttempts?: number;
     maximumInterval?: number;

package/build/types/hook.d.ts

@@ -21,6 +21,7 @@ type HookSignal = {
     topic: string;
     resolved: string;
     jobId: string;
+    expire: number;
 };
 interface HookInterface {
     (topic: string, data: {

package/build/types/hotmesh.d.ts

@@ -76,6 +76,7 @@ type HotMeshGraph = {
     subscribes: string;
     publishes?: string;
     expire?: number;
+    persistent?: boolean;
     output?: {
         schema: Record<string, any>;
     };

package/build/types/job.d.ts

@@ -31,12 +31,12 @@ type JobMetadata = {
     trc: string;
     err?: string;
     expire?: number;
+    persistent?: boolean;
 };
 type ExtensionType = {
     search?: StringStringType;
     marker?: StringStringType;
     pending?: number;
-    statusThreshold?: number;
 };
 type JobStatus = number;
 type JobState = {

package/build/types/meshdata.d.ts

@@ -15,7 +15,6 @@ export type CallOptions = {
     pending?: number;
 };
 export type ConnectOptions = {
-    ttl?: string;
     taskQueue?: string;
     prefix?: string;
     namespace?: string;
@@ -56,4 +55,5 @@ export type MeshDataWorkflowOptions = {
     pending?: number;
     expire?: number;
     signalIn?: boolean;
+    persistent?: boolean;
 };

package/build/types/meshflow.d.ts

@@ -27,6 +27,7 @@ type WorkflowContext = {
     workflowSpan: string;
     raw: StreamData;
     connection: Connection;
+    expire?: number;
 };
 export type WorkflowSearchSchema = Record<string, {
     type: 'TEXT' | 'NUMERIC' | 'TAG';
@@ -68,6 +69,7 @@ type WorkflowOptions = {
     marker?: StringStringType;
     config?: WorkflowConfig;
     expire?: number;
+    persistent?: boolean;
     signalIn?: boolean;
     await?: boolean;
     pending?: number;
@@ -102,6 +104,7 @@ type WorkflowDataType = {
     workflowDimension?: string;
     originJobId?: string;
     canRetry?: boolean;
+    expire?: number;
 };
 type ConnectionConfig = {
     class: Partial<RedisClass>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.2.4",
+  "version": "0.3.1",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/types/activity.ts

@@ -28,7 +28,9 @@ interface BaseActivity {
   telemetry?: Record<string, any>;
   emit?: boolean; //if true, the activity will emit a message to the `publishes` topic immediately before transitioning to adjacent activities
   sleep?: number; //@pipe /in seconds
-  expire?: number; //-1 forever; 0 persists the flow until the parent flow that expired it is dismissed; 15 seconds is the default
+  expire?: number; //-1 forever; 0 persists the flow until the parent flow that expired it is dismissed; 15 seconds is the default (copied from the YAML at compile time)
+  persistent?: boolean; //if true, the job will persist beyond completion (copied from the YAML at compile time)
+  persist?: boolean; //when true, the activity will emit the job completed event and start the `persistence countdown`, using the 'expire' value
   retry?: StreamRetryPolicy;
   cycle?: boolean; //if true, the `notary` will leave leg 2 open, so it can be re/cycled
   collationInt?: number; //compiler

package/types/error.ts

@@ -4,6 +4,7 @@ export type MeshFlowChildErrorType = {
   backoffCoefficient?: number;
   index: number;
   expire?: number;
+  persistent?: boolean;
   signalIn?: boolean;
   maximumAttempts?: number;
   maximumInterval?: number;

package/types/hook.ts

@@ -22,7 +22,12 @@ interface HookRules {
   [eventName: string]: HookRule[];
 }
 
-type HookSignal = { topic: string; resolved: string; jobId: string };
+type HookSignal = {
+  topic: string;
+  resolved: string;
+  jobId: string;
+  expire: number;
+};
 
 interface HookInterface {
   (topic: string, data: { [key: string]: any; id: string }): Promise<void>;

package/types/hotmesh.ts

@@ -88,17 +88,52 @@ type HotMeshConfig = {
 };
 
 type HotMeshGraph = {
+  /**
+   * the unique topic that the graph subscribes to, creating one
+   * job for each idempotent message that is received
+   */
   subscribes: string;
+  /**
+   * the unique topic that the graph publishes/emits to when the job completes
+   */
   publishes?: string;
+  /**
+   * the number of seconds that the completed job should be
+   * left in the store before it is deleted
+   */
   expire?: number;
+  /**
+   * if the graph is reentrant and has open activities, the
+   * `persistent` flag will emit the job completed  event.
+   * This allows the 'main' thread/trigger that started the job to
+   * signal to subscribers (or the parent) that the job
+   * is 'done', while still leaving the job in a
+   * state that allows for reentry (such as cyclical hooks).
+   */
+  persistent?: boolean;
+  /**
+   * the schema for the output of the graph
+   */
   output?: {
     schema: Record<string, any>;
   };
+  /**
+   * the schema for the input of the graph
+   */
   input?: {
     schema: Record<string, any>;
   };
+  /**
+   * the activities that define the graph
+   */
   activities: Record<string, any>;
+  /**
+   * the transitions that define how activities are connected
+   */
   transitions?: Record<string, any>;
+  /**
+   * the reentrant hook rules that define how to reenter a running graph
+   */
   hooks?: HookRules;
 };
 

package/types/job.ts

@@ -63,6 +63,7 @@ type JobMetadata = {
   /** GMT updated //job_updated */
   ju: string;
 
+  /** job status semaphore */
   js: JobStatus;
 
   /** activity_type */
@@ -82,6 +83,9 @@ type JobMetadata = {
 
   /** process data expire policy */
   expire?: number;
+
+  /** job persistence (beyond main thread completion) */
+  persistent?: boolean;
 };
 
 /**
@@ -122,20 +126,6 @@ type ExtensionType = {
    * will be added after the job is resumed if relevant.
    */
   pending?: number;
-
-  /**
-   * Workflows that apply a status threshold will be initialized
-   * with a status value of 1m - statusThreshold.
-   *
-   * The value provided should be the count of descendant activities
-   * (those that descend from the trigger) that should be allowed to
-   * remain open once 'done' event is emitted.
-   *
-   * If the job should not be removed from the cache, the `expire` field
-   * should be set to `0`.
-   *
-   */
-  statusThreshold?: number;
 };
 
 /**

package/types/meshdata.ts

@@ -63,10 +63,6 @@ export type CallOptions = {
 };
 
 export type ConnectOptions = {
-  /**
-   * if set to infinity, callers may not override (the function will be durable)
-   */
-  ttl?: string;
   /**
    * the task queue for the connected function for greater specificity
    */
@@ -265,8 +261,16 @@ export type MeshDataWorkflowOptions = {
   expire?: number;
 
   /**
-   * set to false to optimize workflows that do not require a `signal in`
+   * set to false to optimize workflows that do not require a `signal in`.
+   * explicitly set to true to force the workflow to remain open and persistent.
    * @default true
    */
   signalIn?: boolean;
+
+  /**
+   * set to `true` by the system when a workflow is started with a ttl, ensuring
+   * that the workflow will remain active until the ttl expires and the workflow
+   * is scrubbed. This is a system flag and should not be set by the user.
+   */
+  persistent?: boolean;
 };

package/types/meshflow.ts

@@ -113,6 +113,11 @@ type WorkflowContext = {
    * the HotMesh connection configuration (io/redis NPM package reference and login credentials)
    */
   connection: Connection;
+
+  /**
+   * if present, the workflow will delay expiration for the specified number of seconds
+   */
+  expire?: number;
 };
 
 /**
@@ -316,6 +321,11 @@ type WorkflowOptions = {
    */
   expire?: number;
 
+  /**
+   * system flag to indicate that the flow should remain open beyond main method completion while still emitting the 'job done' event
+   */
+  persistent?: boolean;
+
   /**
    * default is true; set to false to optimize workflows that do not require a `signal in`
    */
@@ -417,8 +427,9 @@ type WorkflowDataType = {
   workflowId: string;
   workflowTopic: string;
   workflowDimension?: string; //is present if hook (not main workflow)
-  originJobId?: string; //is present if there is an originating ancestor job (should rename to originJobId)
+  originJobId?: string; //is present if there is an originating ancestor job
   canRetry?: boolean;
+  expire?: number;
 };
 
 type ConnectionConfig = {