@hotmeshio/hotmesh 0.0.51 → 0.0.53

package/build/services/activities/activity.js

@@ -83,22 +83,22 @@ class Activity {
         }
         catch (error) {
             if (error instanceof errors_1.CollationError) {
-                this.logger.info('process-event-inactive-error', { error });
+                this.logger.info('process-event-inactive-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.InactiveJobError) {
-                this.logger.info('process-event-inactive-job-error', { error });
+                this.logger.info('process-event-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.info('process-event-get-job-error', { error });
+                this.logger.info('process-event-get-job-error', { ...error });
                 return;
             }
-            this.logger.error('activity-process-event-error', { error });
+            this.logger.error('activity-process-event-error', { ...error, message: error.message, stack: error.stack, name: error.name });
             telemetry && telemetry.setActivityError(error.message);
             throw error;
         }
@@ -130,6 +130,10 @@ class Activity {
     async processError(telemetry, type) {
         this.bindActivityError(this.data);
         this.adjacencyList = await this.filterAdjacent();
+        if (!this.adjacencyList.length) {
+            this.bindJobError(this.data);
+        }
+        this.mapJobData();
         const multi = this.store.getMulti();
         await this.setState(multi);
         await collator_1.CollatorService.notarizeCompletion(this, multi);
@@ -142,7 +146,7 @@ class Activity {
         const attrs = { 'app.job.jss': jobStatus };
         //adjacencyList membership has already been set at this point (according to activity status)
         const messageIds = await this.transition(this.adjacencyList, jobStatus);
-        if (messageIds.length) {
+        if (messageIds?.length) {
             attrs['app.activity.mids'] = messageIds.join(',');
         }
         telemetry.setActivityAttributes(attrs);
@@ -159,7 +163,31 @@ class Activity {
     mapJobData() {
         if (this.config.job?.maps) {
             const mapper = new mapper_1.MapperService(this.config.job.maps, this.context);
-            this.context.data = mapper.mapRules();
+            const output = mapper.mapRules();
+            if (output) {
+                for (const key in output) {
+                    const f1 = key.indexOf('[');
+                    //keys with array notation suffix `somekey[]` represent
+                    //dynamically-keyed mappings whose `value` must be moved to the output.
+                    //The `value` must be an object with keys appropriate to the
+                    //notation type: `somekey[0] (array)`, `somekey[-] (mark)`, OR `somekey[_] (search)`
+                    if (f1 > -1) {
+                        const amount = key.substring(f1 + 1).split(']')[0];
+                        if (!isNaN(Number(amount))) {
+                            const left = key.substring(0, f1);
+                            output[left] = output[key];
+                            delete output[key];
+                        }
+                        else if (amount === '-' || amount === '_') {
+                            const obj = output[key];
+                            Object.keys(obj).forEach((newKey) => {
+                                output[newKey] = obj[newKey];
+                            });
+                        }
+                    }
+                }
+            }
+            this.context.data = output;
         }
     }
     mapInputData() {
@@ -181,10 +209,21 @@ class Activity {
     async registerTimeout() {
         //set timeout in support of hook and/or duplex
     }
+    /**
+     * Any StreamMessage with a status of ERROR is bound to the activity
+     */
     bindActivityError(data) {
-        //todo: map activity error data into the job error (if defined)
-        //      map job status via: (500: [3**, 4**, 5**], 202: [$pending])
-        this.context.metadata.err = JSON.stringify(data);
+        const md = this.context[this.metadata.aid].output.metadata;
+        md.err = JSON.stringify(this.data);
+        //(temporary...useful for mapping error parts in the app.yaml)
+        md.$error = { ...data, is_stream_error: true };
+    }
+    /**
+     * unhandled activity errors (activities that return an ERROR StreamMessage
+     * status and have no adjacent children to transition to) are bound to the job
+     */
+    bindJobError(data) {
+        this.context.metadata.err = JSON.stringify({ ...data, is_stream_error: true });
     }
     async getTriggerConfig() {
         return await this.store.getSchema(this.config.trigger, await this.engine.getVID());
@@ -238,9 +277,9 @@ class Activity {
         if (this.status === stream_1.StreamStatus.ERROR) {
             self.output.metadata.err = JSON.stringify(this.data);
         }
-        //todo: verify leg2 never overwrites leg1 `ac`
-        self.output.metadata.ac =
-            self.output.metadata.au = (0, utils_1.formatISODate)(new Date());
+        const ts = (0, utils_1.formatISODate)(new Date());
+        self.output.metadata.ac = ts;
+        self.output.metadata.au = ts;
         self.output.metadata.atp = this.config.type;
         if (this.config.subtype) {
             self.output.metadata.stp = this.config.subtype;
@@ -259,6 +298,11 @@ class Activity {
                 state[path] = value;
             }
         }
+        for (let key in this.context?.data ?? {}) {
+            if (key.startsWith('-') || key.startsWith('_')) {
+                state[key] = this.context.data[key];
+            }
+        }
         telemetry_1.TelemetryService.bindJobTelemetryToState(state, this.config, this.context);
     }
     bindActivityState(state) {
@@ -310,9 +354,9 @@ class Activity {
         let { dad, jid } = this.context.metadata;
         const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
         //`state` is a unidimensional hash; context is a tree
-        const [state, status] = await this.store.getState(jid, consumes, dIds);
+        const [state, _status] = await this.store.getState(jid, consumes, dIds);
         this.context = (0, utils_1.restoreHierarchy)(state);
-        this.assertGenerationalId(this.context.metadata.gid, gid);
+        this.assertGenerationalId(this.context?.metadata?.gid, gid);
         this.initDimensionalAddress(dad);
         this.initSelf(this.context);
         this.initPolicies(this.context);
@@ -325,7 +369,7 @@ class Activity {
      */
     assertGenerationalId(jobGID, msgGID) {
         if (msgGID !== jobGID) {
-            throw new errors_1.GenerationalError(jobGID, msgGID, this.context.metadata.jid, this.context.metadata.aid, this.context.metadata.dad);
+            throw new errors_1.GenerationalError(jobGID, msgGID, this.context?.metadata?.jid ?? '', this.context?.metadata?.aid ?? '', this.context?.metadata?.dad ?? '');
         }
     }
     initDimensionalAddress(dad) {
@@ -346,6 +390,11 @@ class Activity {
         if (!self.hook) {
             self.hook = {};
         }
+        if (!self.output.metadata) {
+            self.output.metadata = {};
+        }
+        //prebind the updated timestamp (mappings need the time)
+        self.output.metadata.au = (0, utils_1.formatISODate)(new Date());
         context['$self'] = self;
         context['$job'] = context; //NEVER call STRINGIFY! (now circular)
         return context;

package/build/services/activities/await.js

@@ -2,12 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Await = void 0;
 const errors_1 = require("../../modules/errors");
+const utils_1 = require("../../modules/utils");
 const activity_1 = require("./activity");
 const collator_1 = require("../collator");
+const pipe_1 = require("../pipe");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-const pipe_1 = require("../pipe");
-const utils_1 = require("../../modules/utils");
 class Await extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -40,19 +40,19 @@ class Await extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.InactiveJobError) {
-                this.logger.error('await-inactive-job-error', { error });
+                this.logger.error('await-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.error('await-get-state-error', { error });
+                this.logger.error('await-get-state-error', { ...error });
                 return;
             }
             else {
-                this.logger.error('await-process-error', { error });
+                this.logger.error('await-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;

package/build/services/activities/cycle.d.ts

@@ -1,6 +1,6 @@
 import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
-import { ActivityData, ActivityMetadata, CycleActivity } from '../../types/activity';
+import { Activity } from './activity';
+import { ActivityData, ActivityMetadata, ActivityType, CycleActivity } from '../../types/activity';
 import { JobState } from '../../types/job';
 import { RedisMulti } from '../../types/redis';
 declare class Cycle extends Activity {

package/build/services/activities/cycle.js

@@ -2,10 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Cycle = void 0;
 const errors_1 = require("../../modules/errors");
+const utils_1 = require("../../modules/utils");
 const collator_1 = require("../collator");
 const activity_1 = require("./activity");
 const telemetry_1 = require("../telemetry");
-const utils_1 = require("../../modules/utils");
 class Cycle extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -40,19 +40,19 @@ class Cycle extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.InactiveJobError) {
-                this.logger.error('cycle-inactive-job-error', { error });
+                this.logger.error('cycle-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.error('cycle-get-state-error', { error });
+                this.logger.error('cycle-get-state-error', { ...error });
                 return;
             }
             else {
-                this.logger.error('cycle-process-error', { error });
+                this.logger.error('cycle-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;

package/build/services/activities/hook.js

@@ -39,19 +39,19 @@ class Hook extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.InactiveJobError) {
-                this.logger.error('hook-inactive-job-error', { error });
+                this.logger.error('hook-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.error('hook-get-state-error', { error });
+                this.logger.error('hook-get-state-error', { ...error });
                 return;
             }
             else {
-                this.logger.error('hook-process-error', { error });
+                this.logger.error('hook-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;

package/build/services/activities/interrupt.d.ts

@@ -1,8 +1,8 @@
+import { Activity } from './activity';
 import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
-import { ActivityData, ActivityMetadata, InterruptActivity } from '../../types/activity';
-import { JobInterruptOptions, JobState } from '../../types/job';
 import { TelemetryService } from '../telemetry';
+import { ActivityData, ActivityMetadata, ActivityType, InterruptActivity } from '../../types/activity';
+import { JobInterruptOptions, JobState } from '../../types/job';
 declare class Interrupt extends Activity {
     config: InterruptActivity;
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);

package/build/services/activities/interrupt.js

@@ -2,10 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Interrupt = void 0;
 const errors_1 = require("../../modules/errors");
-const activity_1 = require("./activity");
 const collator_1 = require("../collator");
-const telemetry_1 = require("../telemetry");
+const activity_1 = require("./activity");
 const pipe_1 = require("../pipe");
+const telemetry_1 = require("../telemetry");
 class Interrupt extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -27,19 +27,19 @@ class Interrupt extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.InactiveJobError) {
-                this.logger.error('interrupt-inactive-job-error', { error });
+                this.logger.error('interrupt-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.error('interrupt-get-state-error', { error });
+                this.logger.error('interrupt-get-state-error', { ...error });
                 return;
             }
             else {
-                this.logger.error('interrupt-process-error', { error });
+                this.logger.error('interrupt-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;
@@ -116,6 +116,15 @@ class Interrupt extends activity_1.Activity {
             descend: this.config.descend !== undefined
                 ? pipe_1.Pipe.resolve(this.config.descend, this.context)
                 : undefined,
+            code: this.config.code !== undefined
+                ? pipe_1.Pipe.resolve(this.config.code, this.context)
+                : undefined,
+            expire: this.config.expire !== undefined
+                ? pipe_1.Pipe.resolve(this.config.expire, this.context)
+                : undefined,
+            stack: this.config.stack !== undefined
+                ? pipe_1.Pipe.resolve(this.config.stack, this.context)
+                : undefined,
         };
     }
     async interrupt() {

package/build/services/activities/signal.d.ts

@@ -1,6 +1,6 @@
-import { Activity, ActivityType } from './activity';
+import { Activity } from './activity';
 import { EngineService } from '../engine';
-import { ActivityData, ActivityMetadata, SignalActivity } from '../../types/activity';
+import { ActivityData, ActivityMetadata, ActivityType, SignalActivity } from '../../types/activity';
 import { JobState } from '../../types/job';
 declare class Signal extends Activity {
     config: SignalActivity;

package/build/services/activities/signal.js

@@ -48,19 +48,19 @@ class Signal extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.InactiveJobError) {
-                this.logger.error('signal-inactive-job-error', { error });
+                this.logger.error('signal-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.error('signal-get-state-error', { error });
+                this.logger.error('signal-get-state-error', { ...error });
                 return;
             }
             else {
-                this.logger.error('signal-process-error', { error });
+                this.logger.error('signal-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;

package/build/services/activities/trigger.js

@@ -47,10 +47,10 @@ class Trigger extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.DuplicateJobError) {
-                this.logger.error('duplicate-job-error', { error });
+                this.logger.error('duplicate-job-error', { job_id: error.jobId });
             }
             else {
-                this.logger.error('trigger-process-error', { error });
+                this.logger.error('trigger-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;
@@ -66,7 +66,16 @@ class Trigger extends activity_1.Activity {
     }
     async execAdjacentParent() {
         if (this.context.metadata.px) {
-            await this.engine.execAdjacentParent(this.context, { metadata: this.context.metadata, data: { job_id: this.context.metadata.jid } });
+            const timestamp = (0, utils_1.formatISODate)(new Date());
+            const jobStartedConfirmationMessage = {
+                metadata: this.context.metadata,
+                data: {
+                    job_id: this.context.metadata.jid,
+                    jc: timestamp,
+                    ju: timestamp,
+                }
+            };
+            await this.engine.execAdjacentParent(this.context, jobStartedConfirmationMessage);
         }
     }
     createInputContext() {

package/build/services/activities/worker.js

@@ -2,11 +2,11 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Worker = void 0;
 const errors_1 = require("../../modules/errors");
+const utils_1 = require("../../modules/utils");
 const activity_1 = require("./activity");
 const collator_1 = require("../collator");
-const telemetry_1 = require("../telemetry");
 const pipe_1 = require("../pipe");
-const utils_1 = require("../../modules/utils");
+const telemetry_1 = require("../telemetry");
 class Worker extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -39,19 +39,19 @@ class Worker extends activity_1.Activity {
         }
         catch (error) {
             if (error instanceof errors_1.InactiveJobError) {
-                this.logger.error('await-inactive-job-error', { error });
+                this.logger.error('await-inactive-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GenerationalError) {
-                this.logger.info('process-event-generational-job-error', { error });
+                this.logger.info('process-event-generational-job-error', { ...error });
                 return;
             }
             else if (error instanceof errors_1.GetStateError) {
-                this.logger.error('worker-get-state-error', { error });
+                this.logger.error('worker-get-state-error', { ...error });
                 return;
             }
             else {
-                this.logger.error('worker-process-error', { error });
+                this.logger.error('worker-process-error', { ...error });
             }
             telemetry.setActivityError(error.message);
             throw error;

package/build/services/compiler/deployer.js

@@ -6,6 +6,7 @@ const utils_1 = require("../../modules/utils");
 const collator_1 = require("../collator");
 const serializer_1 = require("../serializer");
 const pipe_1 = require("../pipe");
+const validator_1 = require("./validator");
 const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
 const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
 const DEFAULT_RANGE_SIZE = DEFAULT_METADATA_RANGE_SIZE + DEFAULT_DATA_RANGE_SIZE;
@@ -185,6 +186,9 @@ class Deployer {
                         //DAGs have one parent; easy to optimize for
                         graph.activities[to].parent = fromActivity;
                     }
+                    //temporarily bind the transitions to the parent activity,
+                    // so the consumer/producer registrar picks up the bindings
+                    graph.activities[fromActivity].transitions = toTransitions;
                 }
             }
         }
@@ -241,9 +245,30 @@ class Deployer {
                         traverse(obj[key], newPath);
                     }
                     else {
-                        const finalPath = `data/${[...path, key].join('/')}`;
-                        if (!result.includes(finalPath)) {
-                            result.push(finalPath);
+                        //wildcard mapping (e.g., 'friends[25]')
+                        //when this is resolved, it will be expanded to
+                        //`'friends/0', ..., 'friends/24'`, providing 25 dynamic
+                        //slots in the flow's output data
+                        const pathName = [...path, key].join('/');
+                        if (!pathName.includes('[')) {
+                            const finalPath = `data/${pathName}`;
+                            if (!result.includes(finalPath)) {
+                                result.push(finalPath);
+                            }
+                        }
+                        else {
+                            const [left, right] = pathName.split('[');
+                            //check if this variable isLiteralKeyType (#, -, or _)
+                            const [amount, _] = right.split(']');
+                            if (!isNaN(parseInt(amount))) {
+                                //loop to create all possible paths (0 to amount)
+                                for (let i = 0; i < parseInt(amount); i++) {
+                                    const finalPath = `data/${left}/${i}`;
+                                    if (!result.includes(finalPath)) {
+                                        result.push(finalPath);
+                                    }
+                                }
+                            } //else ignore (amount might be '-' or '_')  `-` is marker data;   `_` is job data;
                         }
                     }
                 }
@@ -271,7 +296,7 @@ class Deployer {
                 if (typeof obj[key] === 'string') {
                     const stringValue = obj[key];
                     const dynamicMappingRuleMatch = stringValue.match(/^\{[^@].*}$/);
-                    if (dynamicMappingRuleMatch) {
+                    if (dynamicMappingRuleMatch && !validator_1.Validator.CONTEXT_VARS.includes(stringValue)) {
                         if (stringValue.split('.')[1] !== 'input') {
                             dynamicMappingRules.push(stringValue);
                             consumes.push(stringValue);
@@ -340,7 +365,10 @@ class Deployer {
         for (const graph of graphs) {
             const activities = graph.activities;
             for (const activityKey in activities) {
-                activitySchemas[activityKey] = activities[activityKey];
+                const target = activities[activityKey];
+                //remove transitions; no longer necessary for runtime
+                delete target.transitions;
+                activitySchemas[activityKey] = target;
             }
         }
         await this.store.setSchemas(activitySchemas, this.getVID());

package/build/services/compiler/validator.d.ts

@@ -8,6 +8,7 @@ declare class Validator {
     mappingStatements: MappingStatements;
     store: StoreService<RedisClient, RedisMulti> | null;
     static SYS_VARS: string[];
+    static CONTEXT_VARS: string[];
     constructor(manifest: HotMeshManifest);
     /**
      * validate the manifest file
@@ -19,6 +20,7 @@ declare class Validator {
     getMappingStatements(): void;
     validateReferencedActivityIds(): void;
     isFunction(value: string): boolean;
+    isContextVariable(value: string): boolean;
     validateMappingStatements(): void;
     validateTransitions(): void;
     validateTransitionConditions(): void;

package/build/services/compiler/validator.js

@@ -83,7 +83,7 @@ class Validator {
                 if (statement.startsWith('{') && statement.endsWith('}')) {
                     const statementParts = statement.slice(1, -1).split('.');
                     const referencedActivityId = statementParts[0];
-                    if (!(Validator.SYS_VARS.includes(referencedActivityId) || activityIds.includes(referencedActivityId) || this.isFunction(statement))) {
+                    if (!(Validator.SYS_VARS.includes(referencedActivityId) || activityIds.includes(referencedActivityId) || this.isFunction(statement) || this.isContextVariable(statement))) {
                         throw new Error(`Mapping statement references non-existent activity: ${statement}`);
                     }
                 }
@@ -93,6 +93,9 @@ class Validator {
     isFunction(value) {
         return value.startsWith('{@') && pipe_1.Pipe.resolveFunction(value);
     }
+    isContextVariable(value) {
+        return ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'].includes(value);
+    }
     // 1.3) Validate the mapping/@pipe statements are valid
     validateMappingStatements() {
         // Implement the method content
@@ -132,3 +135,4 @@ class Validator {
 }
 exports.Validator = Validator;
 Validator.SYS_VARS = ['$app', '$self', '$graph', '$job'];
+Validator.CONTEXT_VARS = ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'];

package/build/services/durable/client.d.ts

@@ -14,7 +14,13 @@ export declare class ClientService {
      * has not yet been initialized, so this call ensures that the channel
      * exists and is ready to serve as a container for events.
      */
-    createStream: (hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => Promise<void>;
+    static createStream: (hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => Promise<void>;
+    /**
+     * It is possible for a client to invoke a workflow without first
+     * creating the stream. This method will verify that the stream
+     * exists and if not, create it.
+     */
+    static verifyStream: (workflowTopic: string, namespace?: string) => Promise<HotMesh>;
     /**
      * For those deployments with a redis stack backend (with the FT module),
      * this method will configure the search index for the workflow.