@hotmeshio/hotmesh 0.0.51 → 0.0.53

package/services/activities/activity.ts

@@ -143,19 +143,19 @@ class Activity {
       this.transitionAdjacent(multiResponse, telemetry);
     } catch (error) {
       if (error instanceof CollationError) {
-        this.logger.info('process-event-inactive-error', { error });
+        this.logger.info('process-event-inactive-error', { ...error });
         return;
       } else if (error instanceof InactiveJobError) {
-        this.logger.info('process-event-inactive-job-error', { error });
+        this.logger.info('process-event-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.info('process-event-get-job-error', { error });
+        this.logger.info('process-event-get-job-error', {  ...error  });
         return;
       }
-      this.logger.error('activity-process-event-error', { error });
+      this.logger.error('activity-process-event-error', { ...error, message: error.message, stack: error.stack, name: error.name });
       telemetry && telemetry.setActivityError(error.message);
       throw error;
     } finally {
@@ -191,6 +191,10 @@ class Activity {
   async processError(telemetry: TelemetryService, type: string): Promise<MultiResponseFlags> {
     this.bindActivityError(this.data);
     this.adjacencyList = await this.filterAdjacent();
+    if (!this.adjacencyList.length) {
+      this.bindJobError(this.data);
+    }
+    this.mapJobData();
     const multi = this.store.getMulti();
     await this.setState(multi);
     await CollatorService.notarizeCompletion(this, multi);
@@ -205,7 +209,7 @@ class Activity {
     const attrs: StringScalarType = { 'app.job.jss': jobStatus };
     //adjacencyList membership has already been set at this point (according to activity status)
     const messageIds = await this.transition(this.adjacencyList, jobStatus);
-    if (messageIds.length) {
+    if (messageIds?.length) {
       attrs['app.activity.mids'] = messageIds.join(',')
     }
     telemetry.setActivityAttributes(attrs);
@@ -223,7 +227,30 @@ class Activity {
   mapJobData(): void {
     if(this.config.job?.maps) {
       const mapper = new MapperService(this.config.job.maps, this.context);
-      this.context.data = mapper.mapRules();
+      const output = mapper.mapRules();
+      if (output) {
+        for (const key in output) {
+          const f1 = key.indexOf('[');
+          //keys with array notation suffix `somekey[]` represent
+          //dynamically-keyed mappings whose `value` must be moved to the output.
+          //The `value` must be an object with keys appropriate to the
+          //notation type: `somekey[0] (array)`, `somekey[-] (mark)`, OR `somekey[_] (search)`
+          if (f1 > -1) {
+            const amount = key.substring(f1 + 1).split(']')[0];
+            if (!isNaN(Number(amount))) {
+              const left = key.substring(0, f1);
+              output[left] = output[key];
+              delete output[key];
+            } else if (amount === '-' || amount === '_') {
+              const obj = output[key];
+              Object.keys(obj).forEach((newKey) => {
+                output[newKey] = obj[newKey];
+              });
+            }
+          }
+        }
+      }
+      this.context.data = output;
     }
   }
 
@@ -249,10 +276,22 @@ class Activity {
     //set timeout in support of hook and/or duplex
   }
 
+  /**
+   * Any StreamMessage with a status of ERROR is bound to the activity
+   */
   bindActivityError(data: Record<string, unknown>): void {
-    //todo: map activity error data into the job error (if defined)
-    //      map job status via: (500: [3**, 4**, 5**], 202: [$pending])
-    this.context.metadata.err = JSON.stringify(data);
+    const md = this.context[this.metadata.aid].output.metadata;
+    md.err = JSON.stringify(this.data);
+    //(temporary...useful for mapping error parts in the app.yaml)
+    md.$error = { ...data, is_stream_error: true };
+  }
+
+  /**
+   * unhandled activity errors (activities that return an ERROR StreamMessage
+   * status and have no adjacent children to transition to) are bound to the job
+   */
+  bindJobError(data: Record<string, unknown>): void {
+    this.context.metadata.err = JSON.stringify({ ...data, is_stream_error: true });
   }
 
   async getTriggerConfig(): Promise<ActivityType> {
@@ -322,9 +361,9 @@ class Activity {
     if (this.status === StreamStatus.ERROR) {
       self.output.metadata.err = JSON.stringify(this.data);
     }
-    //todo: verify leg2 never overwrites leg1 `ac`
-    self.output.metadata.ac = 
-      self.output.metadata.au = formatISODate(new Date());
+    const ts = formatISODate(new Date());
+    self.output.metadata.ac = ts;
+    self.output.metadata.au = ts;
     self.output.metadata.atp = this.config.type;
     if (this.config.subtype) {
       self.output.metadata.stp = this.config.subtype;
@@ -344,6 +383,11 @@ class Activity {
         state[path] = value;
       }
     }
+    for (let key in this.context?.data ?? {}) {
+      if (key.startsWith('-') || key.startsWith('_')) {
+        state[key] = this.context.data[key];
+      }
+    }
     TelemetryService.bindJobTelemetryToState(state, this.config, this.context);
   }
 
@@ -398,9 +442,9 @@ class Activity {
     let { dad, jid } = this.context.metadata;
     const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
     //`state` is a unidimensional hash; context is a tree
-    const [state, status] = await this.store.getState(jid, consumes, dIds);
+    const [state, _status] = await this.store.getState(jid, consumes, dIds);
     this.context = restoreHierarchy(state) as JobState;
-    this.assertGenerationalId(this.context.metadata.gid, gid);
+    this.assertGenerationalId(this.context?.metadata?.gid, gid);
     this.initDimensionalAddress(dad);
     this.initSelf(this.context);
     this.initPolicies(this.context);
@@ -417,9 +461,9 @@ class Activity {
       throw new GenerationalError(
         jobGID,
         msgGID,
-        this.context.metadata.jid,
-        this.context.metadata.aid,
-        this.context.metadata.dad
+        this.context?.metadata?.jid ?? '',
+        this.context?.metadata?.aid ?? '',
+        this.context?.metadata?.dad ?? ''
       );
     }
   }
@@ -443,6 +487,11 @@ class Activity {
     if (!self.hook) {
       self.hook = { };
     }
+    if (!self.output.metadata) {
+      self.output.metadata = { };
+    }
+    //prebind the updated timestamp (mappings need the time)
+    self.output.metadata.au = formatISODate(new Date());
     context['$self'] = self;
     context['$job'] = context; //NEVER call STRINGIFY! (now circular)
     return context as JobState;

package/services/activities/await.ts

@@ -2,9 +2,11 @@ import {
   GenerationalError,
   GetStateError,
   InactiveJobError } from '../../modules/errors';
+import { guid } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
+import { Pipe } from '../pipe';
 import { TelemetryService } from '../telemetry';
 import {
   ActivityData,
@@ -14,8 +16,6 @@ import {
 import { JobState } from '../../types/job';
 import { MultiResponseFlags, RedisMulti } from '../../types/redis';
 import { StreamData, StreamDataType } from '../../types/stream';
-import { Pipe } from '../pipe';
-import { guid } from '../../modules/utils';
 
 class Await extends Activity {
   config: AwaitActivity;
@@ -60,16 +60,16 @@ class Await extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof InactiveJobError) {
-        this.logger.error('await-inactive-job-error', { error });
+        this.logger.error('await-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.error('await-get-state-error', { error });
+        this.logger.error('await-get-state-error', { ...error });
         return;
       } else {
-        this.logger.error('await-process-error', { error });
+        this.logger.error('await-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;

package/services/activities/cycle.ts

@@ -2,18 +2,19 @@ import {
   GenerationalError,
   GetStateError,
   InactiveJobError } from '../../modules/errors';
+import { guid } from '../../modules/utils';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
+import { Activity } from './activity';
 import {
   ActivityData,
   ActivityMetadata,
+  ActivityType,
   CycleActivity } from '../../types/activity';
 import { JobState } from '../../types/job';
 import { MultiResponseFlags, RedisMulti } from '../../types/redis';
 import { StreamData } from '../../types/stream';
 import { TelemetryService } from '../telemetry';
-import { guid } from '../../modules/utils';
 
 class Cycle extends Activity {
   config: CycleActivity;
@@ -63,16 +64,16 @@ class Cycle extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof InactiveJobError) {
-        this.logger.error('cycle-inactive-job-error', { error });
+        this.logger.error('cycle-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.error('cycle-get-state-error', { error });
+        this.logger.error('cycle-get-state-error', { ...error });
         return;
       } else {
-        this.logger.error('cycle-process-error', { error });
+        this.logger.error('cycle-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;

package/services/activities/hook.ts

@@ -63,16 +63,16 @@ class Hook extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof InactiveJobError) {
-        this.logger.error('hook-inactive-job-error', { error });
+        this.logger.error('hook-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.error('hook-get-state-error', { error });
+        this.logger.error('hook-get-state-error', { ...error });
         return;
       } else {
-        this.logger.error('hook-process-error', { error });
+        this.logger.error('hook-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;

package/services/activities/interrupt.ts

@@ -2,17 +2,18 @@ import {
   GenerationalError,
   GetStateError,
   InactiveJobError } from '../../modules/errors';
+import { CollatorService } from '../collator';
+import { Activity } from './activity';
 import { EngineService } from '../engine';
-import { Activity, ActivityType } from './activity';
+import { Pipe } from '../pipe';
+import { TelemetryService } from '../telemetry';
 import {
   ActivityData,
   ActivityMetadata,
+  ActivityType,
   InterruptActivity } from '../../types/activity';
-import { MultiResponseFlags } from '../../types';
-import { CollatorService } from '../collator';
 import { JobInterruptOptions, JobState } from '../../types/job';
-import { TelemetryService } from '../telemetry';
-import { Pipe } from '../pipe';
+import { MultiResponseFlags } from '../../types/redis';
 
 class Interrupt extends Activity {
   config: InterruptActivity;
@@ -46,16 +47,16 @@ class Interrupt extends Activity {
       }
     } catch (error) {
       if (error instanceof InactiveJobError) {
-        this.logger.error('interrupt-inactive-job-error', { error });
+        this.logger.error('interrupt-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.error('interrupt-get-state-error', { error });
+        this.logger.error('interrupt-get-state-error', { ...error });
         return;
       } else {
-        this.logger.error('interrupt-process-error', { error });
+        this.logger.error('interrupt-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;
@@ -142,6 +143,15 @@ class Interrupt extends Activity {
       descend: this.config.descend !== undefined 
         ? Pipe.resolve(this.config.descend, this.context) 
         : undefined,
+      code: this.config.code !== undefined 
+        ? Pipe.resolve(this.config.code, this.context) 
+        : undefined,
+      expire: this.config.expire !== undefined
+        ? Pipe.resolve(this.config.expire, this.context)
+        : undefined,
+      stack: this.config.stack !== undefined
+        ? Pipe.resolve(this.config.stack, this.context)
+        : undefined,
     };
   }
 

package/services/activities/signal.ts

@@ -2,7 +2,7 @@ import {
   GenerationalError,
   GetStateError,
   InactiveJobError } from '../../modules/errors';
-import { Activity, ActivityType } from './activity';
+import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
 import { MapperService } from '../mapper';
@@ -11,6 +11,7 @@ import { TelemetryService } from '../telemetry';
 import {
   ActivityData,
   ActivityMetadata,
+  ActivityType,
   SignalActivity } from '../../types/activity';
 import { JobState } from '../../types/job';
 import { MultiResponseFlags } from '../../types/redis';
@@ -71,16 +72,16 @@ class Signal extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof InactiveJobError) {
-        this.logger.error('signal-inactive-job-error', { error });
+        this.logger.error('signal-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.error('signal-get-state-error', { error });
+        this.logger.error('signal-get-state-error', { ...error });
         return;
       } else {
-        this.logger.error('signal-process-error', { error });
+        this.logger.error('signal-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;

package/services/activities/trigger.ts

@@ -1,5 +1,8 @@
 import { DuplicateJobError } from '../../modules/errors';
-import { formatISODate, getTimeSeries, guid } from '../../modules/utils';
+import {
+  formatISODate,
+  getTimeSeries,
+  guid } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
@@ -67,9 +70,9 @@ class Trigger extends Activity {
       return this.context.metadata.jid;
     } catch (error) {
       if (error instanceof DuplicateJobError) {
-        this.logger.error('duplicate-job-error', { error });
+        this.logger.error('duplicate-job-error', { job_id: error.jobId });
       } else {
-        this.logger.error('trigger-process-error', { error });
+        this.logger.error('trigger-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;
@@ -86,7 +89,16 @@ class Trigger extends Activity {
 
   async execAdjacentParent() {
     if (this.context.metadata.px) {
-      await this.engine.execAdjacentParent(this.context, {metadata: this.context.metadata, data: { job_id: this.context.metadata.jid }});
+      const timestamp = formatISODate(new Date());
+      const jobStartedConfirmationMessage = {
+        metadata: this.context.metadata,
+        data: {
+          job_id: this.context.metadata.jid,
+          jc: timestamp,
+          ju: timestamp,
+        }
+      };
+      await this.engine.execAdjacentParent(this.context, jobStartedConfirmationMessage);
     }
   }
 

package/services/activities/worker.ts

@@ -2,9 +2,12 @@ import {
   GenerationalError,
   GetStateError,
   InactiveJobError } from '../../modules/errors';
+import { guid } from '../../modules/utils';
 import { Activity } from './activity';
 import { CollatorService } from '../collator';
 import { EngineService } from '../engine';
+import { Pipe } from '../pipe';
+import { TelemetryService } from '../telemetry';
 import {
   ActivityData,
   ActivityMetadata,
@@ -13,9 +16,6 @@ import {
 import { JobState } from '../../types/job';
 import { MultiResponseFlags, RedisMulti } from '../../types/redis';
 import { StreamData} from '../../types/stream';
-import { TelemetryService } from '../telemetry';
-import { Pipe } from '../pipe';
-import { guid } from '../../modules/utils';
 
 class Worker extends Activity {
   config: WorkerActivity;
@@ -61,16 +61,16 @@ class Worker extends Activity {
       return this.context.metadata.aid;
     } catch (error) {
       if (error instanceof InactiveJobError) {
-        this.logger.error('await-inactive-job-error', { error });
+        this.logger.error('await-inactive-job-error', { ...error });
         return;
       } else if (error instanceof GenerationalError) {
-        this.logger.info('process-event-generational-job-error', { error });
+        this.logger.info('process-event-generational-job-error', { ...error });
         return;
       } else if (error instanceof GetStateError) {
-        this.logger.error('worker-get-state-error', { error });
+        this.logger.error('worker-get-state-error', { ...error });
         return;
       } else {
-        this.logger.error('worker-process-error', { error });
+        this.logger.error('worker-process-error', { ...error });
       }
       telemetry.setActivityError(error.message);
       throw error;

package/services/compiler/deployer.ts

@@ -9,6 +9,7 @@ import { HotMeshGraph, HotMeshManifest } from '../../types/hotmesh';
 import { RedisClient, RedisMulti } from '../../types/redis';
 import { StringAnyType, Symbols } from '../../types/serializer';
 import { Pipe } from '../pipe';
+import { Validator } from './validator';
 
 const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
 const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
@@ -201,6 +202,9 @@ class Deployer {
             //DAGs have one parent; easy to optimize for
             graph.activities[to].parent = fromActivity;
           }
+          //temporarily bind the transitions to the parent activity,
+          // so the consumer/producer registrar picks up the bindings
+          graph.activities[fromActivity].transitions = toTransitions;
         }
       }
     }
@@ -257,9 +261,29 @@ class Deployer {
             let newPath = [...path, key];
             traverse(obj[key], newPath);
           } else {
-            const finalPath = `data/${[...path, key].join('/')}`;
-            if (!result.includes(finalPath)) {
-              result.push(finalPath);
+            //wildcard mapping (e.g., 'friends[25]')
+            //when this is resolved, it will be expanded to
+            //`'friends/0', ..., 'friends/24'`, providing 25 dynamic
+            //slots in the flow's output data
+            const pathName = [...path, key].join('/');
+            if (!pathName.includes('[')) {
+              const finalPath = `data/${pathName}`;
+              if (!result.includes(finalPath)) {
+                result.push(finalPath);
+              }
+            } else {
+              const [left, right] = pathName.split('[');
+              //check if this variable isLiteralKeyType (#, -, or _)
+              const [amount, _] = right.split(']');
+              if (!isNaN(parseInt(amount))) {
+                //loop to create all possible paths (0 to amount)
+                for (let i = 0; i < parseInt(amount); i++) {
+                  const finalPath = `data/${left}/${i}`;
+                  if (!result.includes(finalPath)) {
+                    result.push(finalPath);
+                  }
+                }
+              } //else ignore (amount might be '-' or '_')  `-` is marker data;   `_` is job data;
             }
           }
         }
@@ -280,7 +304,7 @@ class Deployer {
       trigger.PRODUCES = results;
     }
   }
-  
+
   resolveMappingDependencies() {
     const dynamicMappingRules: string[] = [];
     //recursive function to descend into the object and find all dynamic mapping rules
@@ -289,7 +313,7 @@ class Deployer {
         if (typeof obj[key] === 'string') {
           const stringValue = obj[key] as string;
           const dynamicMappingRuleMatch = stringValue.match(/^\{[^@].*}$/);
-          if (dynamicMappingRuleMatch) { 
+          if (dynamicMappingRuleMatch && !Validator.CONTEXT_VARS.includes(stringValue)) { 
             if (stringValue.split('.')[1] !== 'input') {
               dynamicMappingRules.push(stringValue);
               consumes.push(stringValue);
@@ -359,7 +383,10 @@ class Deployer {
     for (const graph of graphs) {
       const activities = graph.activities;
       for (const activityKey in activities) {
-        activitySchemas[activityKey] = activities[activityKey];
+        const target = activities[activityKey];
+        //remove transitions; no longer necessary for runtime
+        delete target.transitions;
+        activitySchemas[activityKey] = target;
       }
     }
     await this.store.setSchemas(activitySchemas, this.getVID());

package/services/compiler/validator.ts

@@ -11,7 +11,8 @@ class Validator {
   store: StoreService<RedisClient, RedisMulti> | null = null;
 
   static SYS_VARS = ['$app', '$self', '$graph', '$job'];
-
+  static CONTEXT_VARS = ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'];
+  
   constructor(manifest: HotMeshManifest) {
     this.manifest = manifest;
   }
@@ -92,8 +93,7 @@ class Validator {
         if (statement.startsWith('{') && statement.endsWith('}')) {
           const statementParts = statement.slice(1, -1).split('.');
           const referencedActivityId = statementParts[0];
-  
-          if (!(Validator.SYS_VARS.includes(referencedActivityId) || activityIds.includes(referencedActivityId) || this.isFunction(statement))) {
+          if (!(Validator.SYS_VARS.includes(referencedActivityId) || activityIds.includes(referencedActivityId) || this.isFunction(statement) || this.isContextVariable(statement))) {
             throw new Error(`Mapping statement references non-existent activity: ${statement}`);
           }
         }
@@ -105,6 +105,10 @@ class Validator {
     return value.startsWith('{@') && Pipe.resolveFunction(value);
   }
 
+  isContextVariable(value: string): boolean {
+    return ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'].includes(value);
+  }
+
   // 1.3) Validate the mapping/@pipe statements are valid
   validateMappingStatements() {
     // Implement the method content