@hotmeshio/hotmesh 0.0.3 → 0.0.4

package/README.md

@@ -98,7 +98,7 @@ run().catch((err) => {
 });
 ```
 
->HotMesh delivers durable function execution using a swarm of [distributed engines](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/distributed_orchestration.md). The design  consumes leftover CPU on your microservices to execute workflows without the cost and complexity of a central server.
+>HotMesh delivers durable function execution using a [distributed service mesh](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/distributed_orchestration.md). The design consumes leftover CPU on your microservices to execute workflows without the cost and complexity of a central server/control plane.
 
 ## Advanced Design
 HotMesh's TypeScript SDK is the easiest way to make your functions durable. But if you need full control over your function lifecycles (including high-volume, high-speed use cases), you can use HotMesh's underlying YAML models to optimize your durable workflows. The following model depicts a sequence of activities orchestrated by HotMesh. Any function you associate with a `topic` in your YAML definition is guaranteed to be durable.

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.3",
+    "version": "0.0.4",
     "description": "Durable Workflows",
     "main": "build/index.js",
     "types": "build/index.d.ts",

package/build/services/activities/activity.d.ts

@@ -53,6 +53,8 @@ declare class Activity {
     initSelf(context: StringAnyType): JobState;
     initPolicies(context: JobState): void;
     bindActivityData(type: 'output' | 'hook'): void;
+    resolveDad(): string;
+    resolveAdjacentDad(): string;
     filterAdjacent(): Promise<StreamData[]>;
     transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]>;
 }

package/build/services/activities/activity.js

@@ -100,7 +100,8 @@ class Activity {
             const durationInSeconds = pipe_1.Pipe.resolve(this.config.sleep, this.context);
             const jobId = this.context.metadata.jid;
             const activityId = this.metadata.aid;
-            await this.engine.task.registerTimeHook(jobId, activityId, 'sleep', durationInSeconds);
+            const dId = this.metadata.dad;
+            await this.engine.task.registerTimeHook(jobId, `${activityId}${dId || ''}`, 'sleep', durationInSeconds);
             return jobId;
         }
     }
@@ -158,6 +159,7 @@ class Activity {
             return jobStatus;
         }
         catch (error) {
+            console.error('this error?', error);
             this.logger.error('engine-process-hook-event-error', error);
             telemetry.setActivityError(error.message);
             throw error;
@@ -214,11 +216,10 @@ class Activity {
         }) ?? [];
     }
     bindDimensionalAddress(state) {
-        const { aid, dad } = this.metadata;
-        state[`${aid}/output/metadata/dad`] = dad;
+        const dad = this.resolveDad();
+        state[`${this.metadata.aid}/output/metadata/dad`] = dad;
     }
     async setState(multi) {
-        const { id: appId } = await this.engine.getVID();
         const jobId = this.context.metadata.jid;
         this.bindJobMetadata();
         this.bindActivityMetadata();
@@ -233,7 +234,8 @@ class Activity {
             this.metadata.aid,
             ...presets
         ];
-        return await this.store.setState(state, this.getJobStatus(), jobId, appId, symbolNames, multi);
+        const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], this.resolveDad());
+        return await this.store.setState(state, this.getJobStatus(), jobId, symbolNames, dIds, multi);
     }
     bindJobMetadata() {
         //both legs of the most recently run activity (1 and 2) modify ju (job_updated)
@@ -247,6 +249,7 @@ class Activity {
         if (this.status === stream_1.StreamStatus.ERROR) {
             self.output.metadata.err = JSON.stringify(this.data);
         }
+        //todo: verify leg2 never overwrites leg1 `ac`
         self.output.metadata.ac =
             self.output.metadata.au = (0, utils_1.formatISODate)(new Date());
         self.output.metadata.atp = this.config.type;
@@ -315,10 +318,11 @@ class Activity {
             }
         }
         telemetry_1.TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
-        const { dad, jid } = this.context.metadata;
+        let { dad, jid } = this.context.metadata;
         jobId = jobId || jid;
         //`state` is a flat hash
-        const [state, status] = await this.store.getState(jobId, consumes);
+        const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad);
+        const [state, status] = await this.store.getState(jobId, consumes, dIds);
         //`context` is a tree
         this.context = (0, utils_1.restoreHierarchy)(state);
         this.initDimensionalAddress(dad);
@@ -353,11 +357,25 @@ class Activity {
     bindActivityData(type) {
         this.context[this.metadata.aid][type].data = this.data;
     }
+    resolveDad() {
+        let dad = this.metadata.dad;
+        if (this.adjacentIndex > 0) {
+            //if adjacent index > 0 the activity is cycling; replace last index with cycle index
+            dad = `${dad.substring(0, dad.lastIndexOf(','))},${this.adjacentIndex}`;
+        }
+        return dad;
+    }
+    resolveAdjacentDad() {
+        //concat self and child dimension (all children (leg 1) begin life at 0)
+        return `${this.resolveDad()}${dimension_1.DimensionService.getSeed(0)}`;
+    }
+    ;
     async filterAdjacent() {
         const adjacencyList = [];
         const transitions = await this.store.getTransitions(await this.engine.getVID());
         const transition = transitions[`.${this.metadata.aid}`];
-        const adjacentSuffix = dimension_1.DimensionService.getSeed(this.adjacentIndex);
+        //resolve the dimensional address for adjacent children
+        const adjacentDad = this.resolveAdjacentDad();
         if (transition) {
             for (const toActivityId in transition) {
                 const transitionRule = transition[toActivityId];
@@ -365,7 +383,7 @@ class Activity {
                     adjacencyList.push({
                         metadata: {
                             jid: this.context.metadata.jid,
-                            dad: `${this.metadata.dad}${adjacentSuffix}`,
+                            dad: adjacentDad,
                             aid: toActivityId,
                             spn: this.context['$self'].output.metadata?.l2s,
                             trc: this.context.metadata.trc,

package/build/services/activities/worker.js

@@ -42,6 +42,7 @@ class Worker extends activity_1.Activity {
                 this.logger.error('worker-get-state-error', error);
             }
             else {
+                console.error(error);
                 this.logger.error('worker-process-error', error);
             }
             telemetry.setActivityError(error.message);

package/build/services/collator/index.d.ts

@@ -1,10 +1,11 @@
-import { RedisMulti } from "../../types/redis";
-import { CollationStage } from "../../types/collator";
-import { ActivityDuplex } from "../../types/activity";
-import { HotMeshGraph } from "../../types/hotmesh";
-import { Activity } from "../activities/activity";
+import { RedisMulti } from '../../types/redis';
+import { CollationStage } from '../../types/collator';
+import { ActivityDuplex } from '../../types/activity';
+import { HotMeshGraph } from '../../types/hotmesh';
+import { Activity } from '../activities/activity';
 declare class CollatorService {
     static targetLength: number;
+    static getDimensionalAddress(activity: Activity): Record<string, string>;
     static notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number>;
     static authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
     static notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
@@ -17,6 +18,7 @@ declare class CollatorService {
     static isInactive(num: number): boolean;
     static isPrimed(amount: number, leg: ActivityDuplex): boolean;
     static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
+    static getDimensionsById(ancestors: string[], dad: string): Record<string, string>;
     /**
      * All non-trigger activities are assigned a status seed by their parent
      */

package/build/services/collator/index.js

@@ -4,9 +4,18 @@ exports.CollatorService = void 0;
 const errors_1 = require("../../modules/errors");
 const collator_1 = require("../../types/collator");
 class CollatorService {
+    static getDimensionalAddress(activity) {
+        let dad = activity.context.metadata.dad || activity.metadata.dad;
+        //todo: unsure about this reset
+        // if (dad && activity.leg === 2) {
+        //   console.log('setting dad index back to 0=>', dad);
+        //   dad = `${dad.substring(0, dad.lastIndexOf(','))},0`;
+        // }
+        return CollatorService.getDimensionsById([...activity.config.ancestors, activity.metadata.aid], dad);
+    }
     static async notarizeEntry(activity, multi) {
         //decrement by -100_000_000_000_000
-        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100000000000000, multi);
+        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100000000000000, this.getDimensionalAddress(activity), multi);
         this.verifyInteger(amount, 1, 'enter');
         return amount;
     }
@@ -14,30 +23,30 @@ class CollatorService {
     static async authorizeReentry(activity, multi) {
         //set second digit to 8, allowing for re-entry
         //decrement by -10_000_000_000_000
-        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10000000000000, multi);
+        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10000000000000, this.getDimensionalAddress(activity), multi);
         //this.verifyInteger(amount, 1, 'exit');
         return amount;
     }
     static async notarizeEarlyCompletion(activity, multi) {
         //initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd and 3rd digits to deactivate the activity
-        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000001 - 11000000000000, multi);
+        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000001 - 11000000000000, this.getDimensionalAddress(activity), multi);
     }
     ;
     static async notarizeReentry(activity, multi) {
         //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
-        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000000, multi);
+        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000000, this.getDimensionalAddress(activity), multi);
         this.verifyInteger(amount, 2, 'enter');
         return amount;
     }
     ;
     static async notarizeContinuation(activity, multi) {
         //keep open; actualize the leg2 dimension (+1)
-        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, multi);
+        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, this.getDimensionalAddress(activity), multi);
     }
     ;
     static async notarizeCompletion(activity, multi) {
         //close out; actualize leg2 dimension (+1) and decrement the 3rd digit (-1_000_000_000_000)
-        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - 1000000000000, multi);
+        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - 1000000000000, this.getDimensionalAddress(activity), multi);
     }
     ;
     static getDigitAtIndex(num, targetDigitIndex) {
@@ -106,6 +115,20 @@ class CollatorService {
             throw new errors_1.CollationError(amount, leg, stage, faultType);
         }
     }
+    static getDimensionsById(ancestors, dad) {
+        //ancestors is an ordered list of all ancestors, starting with the trigger (['t1', 'a1', 'a2'])
+        //dad is the dimensional address of the ancestors list (',0,5,3')
+        //loop through the ancestors list and create a map of the ancestor to the dimensional address.
+        //return { 't1': ',0', 'a1': ',0,5', 'a1': ',0,5,3', $ADJACENT: ',0,5,3,0' };
+        // `adjacent` is a special key that is used to track the dimensional address of adjacent activities
+        const map = { '$ADJACENT': `${dad},0` };
+        let dadStr = dad;
+        ancestors.reverse().forEach((ancestor) => {
+            map[ancestor] = dadStr;
+            dadStr = dadStr.substring(0, dadStr.lastIndexOf(','));
+        });
+        return map;
+    }
     /**
      * All non-trigger activities are assigned a status seed by their parent
      */

package/build/services/compiler/deployer.d.ts

@@ -1,7 +1,7 @@
 import { StoreService } from '../store';
-import { HotMeshGraph, HotMeshManifest } from "../../types/hotmesh";
-import { RedisClient, RedisMulti } from "../../types/redis";
-import { Symbols } from "../../types/serializer";
+import { HotMeshGraph, HotMeshManifest } from '../../types/hotmesh';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { Symbols } from '../../types/serializer';
 declare class Deployer {
     manifest: HotMeshManifest | null;
     store: StoreService<RedisClient, RedisMulti> | null;
@@ -12,6 +12,7 @@ declare class Deployer {
         version: string;
     };
     generateSymKeys(): Promise<void>;
+    bindSelf(consumes: Record<string, string[]>, produces: string[], activityId: string): void;
     bindSymbols(startIndex: number, maxIndex: number, existingSymbols: Symbols, prefix: string, produces: string[]): Symbols;
     copyJobSchemas(): void;
     bindBackRefs(): void;

package/build/services/compiler/deployer.js

@@ -53,6 +53,7 @@ class Deployer {
             for (const [activityId, activity] of Object.entries(graph.activities)) {
                 const [lower, upper, symbols] = await this.store.reserveSymbolRange(activityId, DEFAULT_RANGE_SIZE, 'ACTIVITY');
                 const prefix = `${activityId}/`; //activity meta/data is namespaced
+                this.bindSelf(activity.consumes, activity.produces, activityId);
                 const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, activity.produces);
                 if (Object.keys(newSymbols).length) {
                     await this.store.addSymbols(activityId, newSymbols);
@@ -60,6 +61,19 @@ class Deployer {
             }
         }
     }
+    bindSelf(consumes, produces, activityId) {
+        //bind self-referential mappings
+        for (const selfId of [activityId, '$self']) {
+            const selfConsumes = consumes[selfId];
+            if (selfConsumes) {
+                for (const path of selfConsumes) {
+                    if (!produces.includes(path)) {
+                        produces.push(path);
+                    }
+                }
+            }
+        }
+    }
     bindSymbols(startIndex, maxIndex, existingSymbols, prefix, produces) {
         let newSymbols = {};
         let currentSymbols = { ...existingSymbols };

package/build/services/dimension/index.d.ts

@@ -1,4 +1,4 @@
-import { HotMeshGraph } from "../../types/hotmesh";
+import { HotMeshGraph } from '../../types/hotmesh';
 declare class DimensionService {
     static targetLength: number;
     /**

package/build/services/durable/handle.d.ts

@@ -1,4 +1,4 @@
-import { HotMeshService as HotMesh } from "../hotmesh";
+import { HotMeshService as HotMesh } from '../hotmesh';
 export declare class WorkflowHandleService {
     hotMesh: HotMesh;
     workflowTopic: string;

package/build/services/engine/index.d.ts

@@ -58,7 +58,7 @@ declare class EngineService {
     hasParentJob(context: JobState): boolean;
     resolveError(metadata: JobMetadata): StreamError | undefined;
     scrub(jobId: string): Promise<void>;
-    hook(topic: string, data: JobData): Promise<JobStatus | void>;
+    hook(topic: string, data: JobData, dad?: string): Promise<JobStatus | void>;
     hookTime(jobId: string, activityId: string): Promise<JobStatus | void>;
     hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
     pub(topic: string, data: JobData, context?: JobState): Promise<string>;

package/build/services/engine/index.js

@@ -317,21 +317,37 @@ class EngineService {
         await this.store.scrub(jobId);
     }
     // ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
-    async hook(topic, data) {
+    async hook(topic, data, dad) {
         const hookRule = await this.storeSignaler.getHookRule(topic);
+        const [aid, schema] = await this.getSchema(`.${hookRule.to}`);
+        if (!dad) {
+            //assume dimensional address is singular (0)
+            // for ancestors and self if not provided
+            // todo: register
+            dad = ',0'.repeat(schema.ancestors.length + 1);
+        }
         const streamData = {
             type: stream_2.StreamDataType.WEBHOOK,
-            metadata: { aid: `${hookRule.to}`, topic },
+            metadata: {
+                //jid is unknown at this point; will be resolved using the data
+                aid,
+                dad,
+                topic
+            },
             data,
         };
         await this.streamSignaler.publishMessage(null, streamData);
     }
     async hookTime(jobId, activityId) {
+        //the activityid is concatenated with its dimensional address (dad); split to resolve
+        const [aid, ...dimensions] = activityId.split(',');
+        const dad = `,${dimensions.join(',')}`;
         const streamData = {
             type: stream_2.StreamDataType.TIMEHOOK,
             metadata: {
                 jid: jobId,
-                aid: activityId,
+                aid,
+                dad,
             },
             data: { timestamp: Date.now() },
         };
@@ -484,12 +500,13 @@ class EngineService {
     //      (e.g, if {dimensions:true}, use hscan to deliver
     //      the full set of dimensional job data)
     async getState(topic, jobId) {
-        const { id: appId } = await this.getVID();
         const jobSymbols = await this.store.getSymbols(`$${topic}`);
         const consumes = {
             [`$${topic}`]: Object.keys(jobSymbols)
         };
-        const output = await this.store.getState(jobId, consumes);
+        //job data exists at the 'zero' dimension; pass an empty object
+        const dIds = {};
+        const output = await this.store.getState(jobId, consumes, dIds);
         if (!output) {
             throw new Error(`not found ${jobId}`);
         }

package/build/services/hotmesh/index.d.ts

@@ -37,7 +37,7 @@ declare class HotMeshService {
     getIds(topic: string, query: JobStatsInput, queryFacets?: any[]): Promise<IdsResponse>;
     resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
     scrub(jobId: string): Promise<void>;
-    hook(topic: string, data: JobData): Promise<JobStatus | void>;
+    hook(topic: string, data: JobData, dad?: string): Promise<JobStatus | void>;
     hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
     stop(): Promise<void>;
     compress(terms: string[]): Promise<boolean>;

package/build/services/hotmesh/index.js

@@ -116,9 +116,9 @@ class HotMeshService {
         await this.engine?.scrub(jobId);
     }
     // ****** `HOOK` ACTIVITY RE-ENTRY POINT ******
-    async hook(topic, data) {
+    async hook(topic, data, dad) {
         //return collation int
-        return await this.engine?.hook(topic, data);
+        return await this.engine?.hook(topic, data, dad);
     }
     async hookAll(hookTopic, data, query, queryFacets = []) {
         return await this.engine?.hookAll(hookTopic, data, query, queryFacets);

package/build/services/serializer/index.d.ts

@@ -1,3 +1,4 @@
+import { Consumes } from '../../types/activity';
 import { StringStringType, StringAnyType, SymbolMap, SymbolMaps, SymbolSets, Symbols } from '../../types/serializer';
 export declare const MDATA_SYMBOLS: {
     SLOTS: number;
@@ -15,12 +16,16 @@ export declare const MDATA_SYMBOLS: {
     };
 };
 export declare class SerializerService {
+    dIds: StringStringType;
     symKeys: SymbolMaps;
     symReverseKeys: SymbolMaps;
     symValMaps: SymbolMap;
     symValReverseMaps: SymbolMap;
     constructor();
-    resetSymbols(symKeys: SymbolSets, symVals: Symbols): void;
+    abbreviate(consumes: Consumes, symbolNames: string[], fields?: string[]): string[];
+    resolveDimensionalIndex(path: string): string;
+    isJobPath(path: string): boolean;
+    resetSymbols(symKeys: SymbolSets, symVals: Symbols, dIds: StringStringType): void;
     getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap;
     getReverseValueMap(valueMap: SymbolMap): SymbolMap;
     static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols, proposedValues: Set<string>): Symbols;

package/build/services/serializer/index.js

@@ -20,9 +20,45 @@ exports.MDATA_SYMBOLS = {
 };
 class SerializerService {
     constructor() {
-        this.resetSymbols({}, {});
+        this.resetSymbols({}, {}, {});
     }
-    resetSymbols(symKeys, symVals) {
+    abbreviate(consumes, symbolNames, fields = []) {
+        for (const symbolName of symbolNames) {
+            const symbolSet = this.symKeys.get(symbolName);
+            const symbolPaths = consumes[symbolName];
+            for (const symbolPath of symbolPaths) {
+                const abbreviation = symbolSet.get(symbolPath);
+                if (abbreviation) {
+                    const dimensionalIndex = this.resolveDimensionalIndex(symbolPath);
+                    fields.push(`${abbreviation}${dimensionalIndex}`);
+                }
+                else {
+                    fields.push(symbolPath);
+                }
+            }
+        }
+        return fields;
+    }
+    resolveDimensionalIndex(path) {
+        if (this.isJobPath(path)) {
+            return '';
+        }
+        else {
+            const [activityId] = path.split('/');
+            if (activityId in this.dIds) {
+                return this.dIds[activityId];
+            }
+            else if ('$ADJACENT' in this.dIds) {
+                //else=> pre-authorizing adjacent activity entry
+                return this.dIds['$ADJACENT'];
+            }
+            return ',0';
+        }
+    }
+    isJobPath(path) {
+        return path.startsWith('data/') || path.startsWith('metadata/');
+    }
+    resetSymbols(symKeys, symVals, dIds) {
         this.symKeys = new Map();
         this.symReverseKeys = new Map();
         for (const id in symKeys) {
@@ -30,6 +66,7 @@ class SerializerService {
         }
         this.symValMaps = new Map(Object.entries(symVals));
         this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
+        this.dIds = dIds;
     }
     getReverseKeyMap(keyMap, id) {
         let map = this.symReverseKeys.get(id);
@@ -70,24 +107,22 @@ class SerializerService {
         if (this.symKeys.size === 0) {
             return document;
         }
-        let result = { ...document };
-        const compressWithMap = (abbreviationMap) => {
-            for (let key in result) {
-                let safeKey = abbreviationMap.get(key) || key;
-                let value = result[key];
-                let safeValue = abbreviationMap.get(value) || value;
-                if (safeKey !== key || safeValue !== value) {
-                    result[safeKey] = safeValue;
-                    if (safeKey !== key) {
-                        delete result[key];
-                    }
+        let source = { ...document };
+        let result = {};
+        const compressWithMap = (abbreviationMap, id) => {
+            for (let key in source) {
+                if (key.startsWith(`${id}/`) || (id.startsWith('$') && ['data', 'metadata'].includes(key.split('/')[0]))) {
+                    const dimensionalIndex = this.resolveDimensionalIndex(key);
+                    let shortKey = abbreviationMap.get(key) || key;
+                    const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
+                    result[shortDimensionalKey] = source[key];
                 }
             }
         };
         for (let id of ids) {
             const abbreviationMap = this.symKeys.get(id);
             if (abbreviationMap) {
-                compressWithMap(abbreviationMap);
+                compressWithMap(abbreviationMap, id);
             }
         }
         return result;
@@ -100,14 +135,12 @@ class SerializerService {
         const inflateWithMap = (abbreviationMap, id) => {
             const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
             for (let key in result) {
-                let safeKey = reversedAbbreviationMap.get(key) || key;
-                let value = result[key];
-                let safeValue = reversedAbbreviationMap.get(value) || value;
-                if (safeKey !== key || safeValue !== value) {
-                    result[safeKey] = safeValue;
-                    if (safeKey !== key) {
-                        delete result[key];
-                    }
+                //strip dimensional index from key
+                const shortKey = key.split(',')[0];
+                let longKey = reversedAbbreviationMap.get(shortKey);
+                if (longKey) {
+                    result[longKey] = result[key];
+                    delete result[key];
                 }
             }
         };

package/build/services/signaler/stream.js

@@ -121,6 +121,7 @@ class StreamSignaler {
             output = await callback(input);
         }
         catch (err) {
+            console.error(err);
             this.logger.error(`stream-call-function-error`, { stream, id, err });
             output = this.structureUnhandledError(input, err);
         }

package/build/services/store/index.d.ts

@@ -59,9 +59,9 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     getJobIds(indexKeys: string[], idRange: [number, number]): Promise<IdsData>;
     setStatus(collationKeyStatus: number, jobId: string, appId: string, multi?: U): Promise<any>;
     getStatus(jobId: string, appId: string): Promise<number>;
-    setState({ ...state }: StringAnyType, status: number | null, jobId: string, appId: string, symbolNames: string[], multi?: U): Promise<string>;
-    getState(jobId: string, consumes: Consumes): Promise<[StringAnyType, number] | undefined>;
-    collate(jobId: string, activityId: string, amount: number, multi?: U): Promise<number>;
+    setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
+    getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
+    collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
     setStateNX(jobId: string, appId: string): Promise<boolean>;
     getSchema(activityId: string, appVersion: AppVID): Promise<ActivityType>;
     getSchemas(appVersion: AppVID): Promise<Record<string, ActivityType>>;

package/build/services/store/index.js

@@ -374,12 +374,12 @@ class StoreService {
         const status = await this.redisClient[this.commands.hget](jobKey, ':');
         return Number(status);
     }
-    async setState({ ...state }, status, jobId, appId, symbolNames, multi) {
+    async setState({ ...state }, status, jobId, symbolNames, dIds, multi) {
         delete state['metadata/js'];
-        const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
+        const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         const symKeys = await this.getSymbolKeys(symbolNames);
         const symVals = await this.getSymbolValues();
-        this.serializer.resetSymbols(symKeys, symVals);
+        this.serializer.resetSymbols(symKeys, symVals, dIds);
         const hashData = this.serializer.package(state, symbolNames);
         if (status !== null) {
             hashData[':'] = status.toString();
@@ -390,26 +390,13 @@ class StoreService {
         await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
         return jobId;
     }
-    async getState(jobId, consumes) {
-        //todo: refactor into semantic submethods
+    async getState(jobId, consumes, dIds) {
+        //get abbreviated field list (the symbols for the paths)
         const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         const symbolNames = Object.keys(consumes);
         const symKeys = await this.getSymbolKeys(symbolNames);
-        //always fetch the job status (':') when fetching state
-        const fields = [':'];
-        for (const symbolName of symbolNames) {
-            const symbolSet = symKeys[symbolName];
-            const symbolPaths = consumes[symbolName];
-            for (const symbolPath of symbolPaths) {
-                const abbreviation = symbolSet[symbolPath];
-                if (abbreviation) {
-                    fields.push(abbreviation);
-                }
-                else {
-                    fields.push(symbolPath);
-                }
-            }
-        }
+        this.serializer.resetSymbols(symKeys, {}, dIds);
+        const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
         const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
         const jobData = {};
         let atLeast1 = false; //if status field (':') isn't present assume 404
@@ -421,7 +408,7 @@ class StoreService {
         });
         if (atLeast1) {
             const symVals = await this.getSymbolValues();
-            this.serializer.resetSymbols(symKeys, symVals);
+            this.serializer.resetSymbols(symKeys, symVals, dIds);
             const state = this.serializer.unpackage(jobData, symbolNames);
             let status = 0;
             if (state[':']) {
@@ -432,13 +419,13 @@ class StoreService {
             return [state, status];
         }
     }
-    async collate(jobId, activityId, amount, multi) {
+    async collate(jobId, activityId, amount, dIds, multi) {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         const collationKey = `${activityId}/output/metadata/as`; //activity state
         const symbolNames = [activityId];
         const symKeys = await this.getSymbolKeys(symbolNames);
         const symVals = await this.getSymbolValues();
-        this.serializer.resetSymbols(symKeys, symVals);
+        this.serializer.resetSymbols(symKeys, symVals, dIds);
         const payload = { [collationKey]: amount.toString() };
         const hashData = this.serializer.package(payload, symbolNames);
         const targetId = Object.keys(hashData)[0];

package/build/types/activity.d.ts

@@ -23,6 +23,7 @@ interface BaseActivity {
     subscribes?: string;
     trigger?: string;
     parent?: string;
+    ancestors?: string[];
 }
 interface Measure {
     measure: MetricTypes;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "Durable Workflows",
   "main": "build/index.js",
   "types": "build/index.d.ts",

package/services/activities/activity.ts

@@ -142,7 +142,8 @@ class Activity {
       const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
       const jobId = this.context.metadata.jid;
       const activityId = this.metadata.aid;
-      await this.engine.task.registerTimeHook(jobId, activityId, 'sleep', durationInSeconds);
+      const dId = this.metadata.dad;
+      await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
       return jobId;
     }
   }
@@ -206,6 +207,7 @@ class Activity {
       telemetry.setActivityAttributes(attrs);
       return jobStatus as number;
     } catch (error) {
+      console.error('this error?', error);
       this.logger.error('engine-process-hook-event-error', error);
       telemetry.setActivityError(error.message);
       throw error;
@@ -278,12 +280,11 @@ class Activity {
   }
 
   bindDimensionalAddress(state: StringAnyType) {
-    const { aid, dad } = this.metadata;
-    state[`${aid}/output/metadata/dad`] = dad;
+    const dad = this.resolveDad();
+    state[`${this.metadata.aid}/output/metadata/dad`] = dad;
   }
 
   async setState(multi?: RedisMulti): Promise<string> {
-    const { id: appId } = await this.engine.getVID();
     const jobId = this.context.metadata.jid;
     this.bindJobMetadata();
     this.bindActivityMetadata();
@@ -298,7 +299,8 @@ class Activity {
       this.metadata.aid,
       ...presets
     ];
-    return await this.store.setState(state, this.getJobStatus(), jobId, appId, symbolNames, multi);
+    const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], this.resolveDad());
+    return await this.store.setState(state, this.getJobStatus(), jobId, symbolNames, dIds, multi);
   }
 
   bindJobMetadata(): void {
@@ -314,6 +316,7 @@ class Activity {
     if (this.status === StreamStatus.ERROR) {
       self.output.metadata.err = JSON.stringify(this.data);
     }
+    //todo: verify leg2 never overwrites leg1 `ac`
     self.output.metadata.ac = 
       self.output.metadata.au = formatISODate(new Date());
     self.output.metadata.atp = this.config.type;
@@ -386,10 +389,11 @@ class Activity {
       }
     }
     TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
-    const { dad, jid } = this.context.metadata;
+    let { dad, jid } = this.context.metadata;
     jobId = jobId || jid;
     //`state` is a flat hash
-    const [state, status] = await this.store.getState(jobId, consumes);
+    const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad);
+    const [state, status] = await this.store.getState(jobId, consumes, dIds);
     //`context` is a tree
     this.context = restoreHierarchy(state) as JobState;
     this.initDimensionalAddress(dad);
@@ -429,11 +433,26 @@ class Activity {
     this.context[this.metadata.aid][type].data = this.data;
   }
 
+  resolveDad(): string {
+    let dad = this.metadata.dad;
+    if (this.adjacentIndex > 0) {
+      //if adjacent index > 0 the activity is cycling; replace last index with cycle index
+      dad = `${dad.substring(0, dad.lastIndexOf(','))},${this.adjacentIndex}`
+    }
+    return dad;
+  }
+
+  resolveAdjacentDad(): string {
+    //concat self and child dimension (all children (leg 1) begin life at 0)
+    return `${this.resolveDad()}${DimensionService.getSeed(0)}`;
+  };
+
   async filterAdjacent(): Promise<StreamData[]> {
     const adjacencyList: StreamData[] = [];
     const transitions = await this.store.getTransitions(await this.engine.getVID());
     const transition = transitions[`.${this.metadata.aid}`];
-    const adjacentSuffix = DimensionService.getSeed(this.adjacentIndex);
+    //resolve the dimensional address for adjacent children
+    const adjacentDad = this.resolveAdjacentDad();
     if (transition) {
       for (const toActivityId in transition) {
         const transitionRule: boolean | TransitionRule = transition[toActivityId];
@@ -441,7 +460,7 @@ class Activity {
           adjacencyList.push({
             metadata: {
               jid: this.context.metadata.jid,
-              dad: `${this.metadata.dad}${adjacentSuffix}`,
+              dad: adjacentDad,
               aid: toActivityId,
               spn: this.context['$self'].output.metadata?.l2s,
               trc: this.context.metadata.trc,

package/services/activities/worker.ts

@@ -62,6 +62,7 @@ class Worker extends Activity {
       if (error instanceof GetStateError) {
         this.logger.error('worker-get-state-error', error);
       } else {
+        console.error(error);
         this.logger.error('worker-process-error', error);
       }
       telemetry.setActivityError(error.message);

package/services/collator/index.ts

@@ -1,18 +1,28 @@
-import { CollationError } from "../../modules/errors";
-import { RedisMulti } from "../../types/redis";
-import { CollationFaultType, CollationStage } from "../../types/collator";
-import { ActivityDuplex } from "../../types/activity";
-import { HotMeshGraph } from "../../types/hotmesh";
-import { Activity } from "../activities/activity";
+import { CollationError } from '../../modules/errors';
+import { RedisMulti } from '../../types/redis';
+import { CollationFaultType, CollationStage } from '../../types/collator';
+import { ActivityDuplex } from '../../types/activity';
+import { HotMeshGraph } from '../../types/hotmesh';
+import { Activity } from '../activities/activity';
 
 class CollatorService {
 
   //max int digit count that supports `hincrby`
-  static targetLength = 15; 
+  static targetLength = 15;
+
+  static getDimensionalAddress(activity: Activity): Record<string, string> {
+    let dad = activity.context.metadata.dad || activity.metadata.dad;
+    //todo: unsure about this reset
+    // if (dad && activity.leg === 2) {
+    //   console.log('setting dad index back to 0=>', dad);
+    //   dad = `${dad.substring(0, dad.lastIndexOf(','))},0`;
+    // }
+    return CollatorService.getDimensionsById([...activity.config.ancestors, activity.metadata.aid], dad);
+  }
 
   static async notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number> {
     //decrement by -100_000_000_000_000
-    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100_000_000_000_000, multi);
+    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100_000_000_000_000, this.getDimensionalAddress(activity), multi);
     this.verifyInteger(amount, 1, 'enter');
     return amount;
   };
@@ -20,31 +30,31 @@ class CollatorService {
   static async authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number> {
     //set second digit to 8, allowing for re-entry
     //decrement by -10_000_000_000_000
-    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10_000_000_000_000, multi);
+    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10_000_000_000_000, this.getDimensionalAddress(activity), multi);
     //this.verifyInteger(amount, 1, 'exit');
     return amount;
   }
 
   static async notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
     //initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd and 3rd digits to deactivate the activity
-    return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_001 - 11_000_000_000_000, multi);
+    return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_001 - 11_000_000_000_000, this.getDimensionalAddress(activity), multi);
   };
 
   static async notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number> {
     //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
-    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_000, multi);
+    const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1_000_000, this.getDimensionalAddress(activity), multi);
     this.verifyInteger(amount, 2, 'enter');
     return amount;
   };
 
   static async notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number> {
     //keep open; actualize the leg2 dimension (+1)
-    return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, multi);
+    return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, this.getDimensionalAddress(activity), multi);
   };
 
   static async notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number> {
     //close out; actualize leg2 dimension (+1) and decrement the 3rd digit (-1_000_000_000_000)
-    return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - 1_000_000_000_000, multi);
+    return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - 1_000_000_000_000, this.getDimensionalAddress(activity), multi);
   };
 
   static getDigitAtIndex(num: number, targetDigitIndex: number): number | null {
@@ -112,6 +122,21 @@ class CollatorService {
     }
   }
 
+  static getDimensionsById(ancestors: string[], dad: string): Record<string, string> {
+    //ancestors is an ordered list of all ancestors, starting with the trigger (['t1', 'a1', 'a2'])
+    //dad is the dimensional address of the ancestors list (',0,5,3')
+    //loop through the ancestors list and create a map of the ancestor to the dimensional address.
+    //return { 't1': ',0', 'a1': ',0,5', 'a1': ',0,5,3', $ADJACENT: ',0,5,3,0' };
+    // `adjacent` is a special key that is used to track the dimensional address of adjacent activities
+    const map: Record<string, string> = { '$ADJACENT': `${dad},0` };
+    let dadStr = dad;
+    ancestors.reverse().forEach((ancestor) => {
+      map[ancestor] = dadStr;
+      dadStr = dadStr.substring(0, dadStr.lastIndexOf(','));
+    });
+    return map;
+  }
+
   /**
    * All non-trigger activities are assigned a status seed by their parent
    */

package/services/compiler/deployer.ts

@@ -1,13 +1,13 @@
-import { KeyStoreParams, KeyType } from "../../modules/key";
-import { getSymKey } from "../../modules/utils";
-import { CollatorService } from "../collator";
-import { SerializerService } from "../serializer";
+import { KeyStoreParams, KeyType } from '../../modules/key';
+import { getSymKey } from '../../modules/utils';
+import { CollatorService } from '../collator';
+import { SerializerService } from '../serializer';
 import { StoreService } from '../store';
-import { ActivityType } from "../../types/activity";
-import { HookRule } from "../../types/hook";
-import { HotMeshGraph, HotMeshManifest } from "../../types/hotmesh";
-import { RedisClient, RedisMulti } from "../../types/redis";
-import { StringAnyType, Symbols } from "../../types/serializer";
+import { ActivityType } from '../../types/activity';
+import { HookRule } from '../../types/hook';
+import { HotMeshGraph, HotMeshManifest } from '../../types/hotmesh';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { StringAnyType, Symbols } from '../../types/serializer';
 
 const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
 const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
@@ -63,6 +63,7 @@ class Deployer {
       for (const [activityId, activity] of Object.entries(graph.activities)) {
         const [lower, upper, symbols] = await this.store.reserveSymbolRange(activityId, DEFAULT_RANGE_SIZE, 'ACTIVITY');
         const prefix = `${activityId}/`; //activity meta/data is namespaced
+        this.bindSelf(activity.consumes, activity.produces, activityId);
         const newSymbols = this.bindSymbols(lower, upper, symbols, prefix, activity.produces);
         if (Object.keys(newSymbols).length) {
           await this.store.addSymbols(activityId, newSymbols);
@@ -71,6 +72,20 @@ class Deployer {
     }
   }
 
+  bindSelf(consumes: Record<string, string[]>, produces: string[], activityId: string) {
+    //bind self-referential mappings
+    for (const selfId of [activityId, '$self']) {
+      const selfConsumes = consumes[selfId];
+      if (selfConsumes) {
+        for (const path of selfConsumes) {
+          if (!produces.includes(path)) {
+            produces.push(path);
+          }
+        }
+      }
+    }
+  }
+
   bindSymbols(startIndex: number, maxIndex: number, existingSymbols: Symbols, prefix: string, produces: string[]): Symbols {
     let newSymbols: Symbols = {};
     let currentSymbols: Symbols = {...existingSymbols};

package/services/dimension/index.ts

@@ -1,4 +1,4 @@
-import { HotMeshGraph } from "../../types/hotmesh";
+import { HotMeshGraph } from '../../types/hotmesh';
 
 class DimensionService {
 

package/services/durable/handle.ts

@@ -1,5 +1,5 @@
-import { JobOutput } from "../../types/job";
-import { HotMeshService as HotMesh } from "../hotmesh";
+import { JobOutput } from '../../types/job';
+import { HotMeshService as HotMesh } from '../hotmesh';
 
 export class WorkflowHandleService {
   hotMesh: HotMesh;

package/services/engine/index.ts

@@ -63,6 +63,7 @@ import {
   StreamError,
   StreamRole,
   StreamStatus } from '../../types/stream';
+import { StringStringType } from '../../types';
 
 //wait time to see if a job is complete
 const OTT_WAIT_TIME = 1000;
@@ -408,21 +409,37 @@ class EngineService {
   }
 
   // ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
-  async hook(topic: string, data: JobData): Promise<JobStatus | void> {
+  async hook(topic: string, data: JobData, dad?: string): Promise<JobStatus | void> {
     const hookRule = await this.storeSignaler.getHookRule(topic);
+    const [aid, schema] = await this.getSchema(`.${hookRule.to}`);
+    if (!dad) {
+      //assume dimensional address is singular (0)
+      // for ancestors and self if not provided
+      // todo: register
+      dad = ',0'.repeat(schema.ancestors.length + 1);
+    }
     const streamData: StreamData = {
       type: StreamDataType.WEBHOOK,
-      metadata: { aid: `${hookRule.to}`, topic },
+      metadata: {
+        //jid is unknown at this point; will be resolved using the data
+        aid,
+        dad,
+        topic
+      },
       data,
     };
     await this.streamSignaler.publishMessage(null, streamData);
   }
   async hookTime(jobId: string, activityId: string): Promise<JobStatus | void> {
+    //the activityid is concatenated with its dimensional address (dad); split to resolve
+    const [aid, ...dimensions] = activityId.split(',');
+    const dad = `,${dimensions.join(',')}`;
     const streamData: StreamData = {
       type: StreamDataType.TIMEHOOK,
       metadata: {
         jid: jobId,
-        aid: activityId,
+        aid,
+        dad,
       },
       data: { timestamp: Date.now() },
     };
@@ -586,12 +603,13 @@ class EngineService {
   //      (e.g, if {dimensions:true}, use hscan to deliver
   //      the full set of dimensional job data)
   async getState(topic: string, jobId: string): Promise<JobOutput> {
-    const { id: appId } = await this.getVID();
     const jobSymbols = await this.store.getSymbols(`$${topic}`);
     const consumes: Consumes = {
       [`$${topic}`]: Object.keys(jobSymbols)
     }
-    const output = await this.store.getState(jobId, consumes);
+    //job data exists at the 'zero' dimension; pass an empty object
+    const dIds = {} as StringStringType;
+    const output = await this.store.getState(jobId, consumes, dIds);
     if (!output) {
       throw new Error(`not found ${jobId}`);
     }

package/services/hotmesh/index.ts

@@ -161,9 +161,9 @@ class HotMeshService {
   }
 
   // ****** `HOOK` ACTIVITY RE-ENTRY POINT ******
-  async hook(topic: string, data: JobData): Promise<JobStatus | void> {
+  async hook(topic: string, data: JobData, dad?: string): Promise<JobStatus | void> {
     //return collation int
-    return await this.engine?.hook(topic, data);
+    return await this.engine?.hook(topic, data, dad);
   }
   async hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets: string[] = []): Promise<string[]> {
     return await this.engine?.hookAll(hookTopic, data, query, queryFacets);

package/services/mapper/index.ts

@@ -80,5 +80,4 @@ class MapperService {
   }
 }
 
-
 export { MapperService }

package/services/serializer/index.ts

@@ -1,4 +1,5 @@
 import { getSymVal } from '../../modules/utils';
+import { Consumes } from '../../types/activity';
 import {
   StringStringType,
   StringAnyType,
@@ -26,16 +27,53 @@ export const MDATA_SYMBOLS = {
 };
 
 export class SerializerService {
+  dIds: StringStringType;
   symKeys: SymbolMaps;
   symReverseKeys: SymbolMaps;
   symValMaps: SymbolMap;
   symValReverseMaps: SymbolMap;
 
   constructor() {
-    this.resetSymbols({}, {});
+    this.resetSymbols({}, {}, {});
   }
 
-  resetSymbols(symKeys: SymbolSets, symVals: Symbols): void {
+  abbreviate(consumes: Consumes, symbolNames: string[], fields: string[] = []): string[] {
+    for (const symbolName of symbolNames) {
+      const symbolSet = this.symKeys.get(symbolName);
+      const symbolPaths = consumes[symbolName];
+      for (const symbolPath of symbolPaths) {
+        const abbreviation = symbolSet.get(symbolPath);
+        if (abbreviation) {
+          const dimensionalIndex = this.resolveDimensionalIndex(symbolPath);
+          fields.push(`${abbreviation}${dimensionalIndex}`);
+        } else {
+          fields.push(symbolPath);
+        }
+      }
+    }
+    return fields;
+  }
+
+  resolveDimensionalIndex(path: string): string {
+    if (this.isJobPath(path)) {
+      return '';
+    } else {
+      const [activityId] = path.split('/');
+      if (activityId in this.dIds) {
+        return this.dIds[activityId];
+      } else if ('$ADJACENT' in this.dIds) {
+        //else=> pre-authorizing adjacent activity entry
+        return this.dIds['$ADJACENT'];
+      }
+      return ',0';
+    }
+  }
+
+  isJobPath(path: string): boolean {
+    return path.startsWith('data/') || path.startsWith('metadata/');
+  }
+
+  resetSymbols(symKeys: SymbolSets, symVals: Symbols, dIds: StringStringType): void {
     this.symKeys = new Map();
     this.symReverseKeys = new Map();
     for (const id in symKeys) {
@@ -43,6 +81,7 @@ export class SerializerService {
     }
     this.symValMaps = new Map(Object.entries(symVals));
     this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
+    this.dIds = dIds;
   }
 
   getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap {
@@ -87,25 +126,23 @@ export class SerializerService {
     if (this.symKeys.size === 0) {
       return document;
     }
-    let result: StringStringType = { ...document };
+    let source: StringStringType = { ...document };
+    let result: StringStringType = { };
 
-    const compressWithMap = (abbreviationMap: SymbolMap) => {
-      for (let key in result) {
-        let safeKey = abbreviationMap.get(key) || key;
-        let value = result[key];
-        let safeValue = abbreviationMap.get(value) || value;
-        if (safeKey !== key || safeValue !== value) {
-          result[safeKey] = safeValue;
-          if (safeKey !== key) {
-            delete result[key];
+    const compressWithMap = (abbreviationMap: SymbolMap, id: string) => {
+      for (let key in source) {
+        if (key.startsWith(`${id}/`) || (id.startsWith('$') && ['data', 'metadata'].includes(key.split('/')[0]))) {
+          const dimensionalIndex = this.resolveDimensionalIndex(key);
+          let shortKey = abbreviationMap.get(key) || key;
+          const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
+          result[shortDimensionalKey] = source[key];
           }
-        }
       }
     };
     for (let id of ids) {
       const abbreviationMap = this.symKeys.get(id);
       if (abbreviationMap) {
-        compressWithMap(abbreviationMap);
+        compressWithMap(abbreviationMap, id);
       }
     }
     return result;
@@ -120,14 +157,12 @@ export class SerializerService {
     const inflateWithMap = (abbreviationMap: SymbolMap, id: string) => {
       const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
       for (let key in result) {
-        let safeKey = reversedAbbreviationMap.get(key) || key;
-        let value = result[key];
-        let safeValue = reversedAbbreviationMap.get(value) || value;
-        if (safeKey !== key || safeValue !== value) {
-          result[safeKey] = safeValue;
-          if (safeKey !== key) {
-            delete result[key];
-          }
+        //strip dimensional index from key
+        const shortKey = key.split(',')[0];
+        let longKey = reversedAbbreviationMap.get(shortKey);
+        if (longKey) {
+          result[longKey] = result[key];
+          delete result[key];
         }
       }
     };

package/services/signaler/stream.ts

@@ -150,6 +150,7 @@ class StreamSignaler {
     try {
       output = await callback(input);
     } catch (err) {
+      console.error(err);
       this.logger.error(`stream-call-function-error`, { stream, id, err });
       output = this.structureUnhandledError(input, err);
     }

package/services/store/index.ts

@@ -452,12 +452,12 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return Number(status);
   }
 
-  async setState({ ...state }: StringAnyType, status: number | null, jobId: string, appId: string, symbolNames: string[], multi? : U): Promise<string> {
+  async setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi? : U): Promise<string> {
     delete state['metadata/js'];
-    const hashKey = this.mintKey(KeyType.JOB_STATE, { appId, jobId });
+    const hashKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     const symKeys = await this.getSymbolKeys(symbolNames);
     const symVals = await this.getSymbolValues();
-    this.serializer.resetSymbols(symKeys, symVals);
+    this.serializer.resetSymbols(symKeys, symVals, dIds);
 
     const hashData = this.serializer.package(state, symbolNames);
     if (status !== null) {
@@ -469,26 +469,14 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     return jobId;
   }
 
-  async getState(jobId: string, consumes: Consumes): Promise<[StringAnyType, number] | undefined> {
-    //todo: refactor into semantic submethods
+  async getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined> {
+    //get abbreviated field list (the symbols for the paths)
     const key = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     const symbolNames = Object.keys(consumes);
     const symKeys = await this.getSymbolKeys(symbolNames);
+    this.serializer.resetSymbols(symKeys, {}, dIds);
+    const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
 
-    //always fetch the job status (':') when fetching state
-    const fields = [':'];
-    for (const symbolName of symbolNames) {
-      const symbolSet = symKeys[symbolName];
-      const symbolPaths = consumes[symbolName];
-      for (const symbolPath of symbolPaths) {
-        const abbreviation = symbolSet[symbolPath];
-        if (abbreviation) {
-          fields.push(abbreviation);
-        } else {
-          fields.push(symbolPath);
-        }
-      }
-    }
     const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
     const jobData: StringAnyType = {};
     let atLeast1 = false; //if status field (':') isn't present assume 404
@@ -500,7 +488,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     });
     if (atLeast1) {
       const symVals = await this.getSymbolValues();
-      this.serializer.resetSymbols(symKeys, symVals);
+      this.serializer.resetSymbols(symKeys, symVals, dIds);
       const state = this.serializer.unpackage(jobData, symbolNames);
       let status = 0;
       if (state[':']) {
@@ -512,13 +500,13 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
-  async collate(jobId: string, activityId: string, amount: number, multi? : U): Promise<number> {
+  async collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi? : U): Promise<number> {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     const collationKey = `${activityId}/output/metadata/as`; //activity state
     const symbolNames = [activityId];
     const symKeys = await this.getSymbolKeys(symbolNames);
     const symVals = await this.getSymbolValues();
-    this.serializer.resetSymbols(symKeys, symVals);
+    this.serializer.resetSymbols(symKeys, symVals, dIds);
 
     const payload = { [collationKey]: amount.toString() }
     const hashData = this.serializer.package(payload, symbolNames);

package/types/activity.ts

@@ -26,6 +26,7 @@ interface BaseActivity {
   subscribes?: string;                 //compiler
   trigger?: string;                    //compiler
   parent?: string;                     //compiler
+  ancestors?: string[];                //compiler
 }
 
 interface Measure {