@hotmeshio/hotmesh 0.0.42 → 0.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/modules/enums.d.ts +2 -0
- package/build/modules/enums.js +4 -1
- package/build/modules/utils.js +1 -1
- package/build/package.json +1 -1
- package/build/services/activities/trigger.js +7 -1
- package/build/services/durable/client.d.ts +2 -1
- package/build/services/durable/client.js +17 -3
- package/build/services/durable/exporter.d.ts +105 -0
- package/build/services/durable/exporter.js +374 -0
- package/build/services/durable/factory.js +6 -63
- package/build/services/durable/handle.d.ts +4 -0
- package/build/services/durable/handle.js +5 -0
- package/build/services/durable/meshos.js +3 -0
- package/build/services/durable/workflow.js +24 -21
- package/build/services/engine/index.d.ts +6 -1
- package/build/services/engine/index.js +9 -2
- package/build/services/exporter/index.d.ts +46 -0
- package/build/services/exporter/index.js +126 -0
- package/build/services/hotmesh/index.d.ts +4 -1
- package/build/services/hotmesh/index.js +6 -0
- package/build/services/quorum/index.d.ts +5 -2
- package/build/services/quorum/index.js +33 -15
- package/build/services/router/index.d.ts +3 -0
- package/build/services/router/index.js +3 -0
- package/build/services/store/clients/redis.js +1 -0
- package/build/services/store/index.d.ts +7 -3
- package/build/services/store/index.js +62 -12
- package/build/services/task/index.js +5 -1
- package/build/services/worker/index.js +5 -4
- package/build/types/activity.d.ts +6 -1
- package/build/types/exporter.d.ts +51 -0
- package/build/types/exporter.js +8 -0
- package/build/types/hotmesh.d.ts +1 -1
- package/build/types/index.d.ts +1 -0
- package/build/types/quorum.d.ts +1 -0
- package/build/types/task.d.ts +1 -1
- package/modules/enums.ts +4 -0
- package/modules/utils.ts +1 -1
- package/package.json +1 -1
- package/services/activities/trigger.ts +14 -0
- package/services/durable/client.ts +19 -4
- package/services/durable/exporter.ts +408 -0
- package/services/durable/factory.ts +6 -63
- package/services/durable/handle.ts +12 -0
- package/services/durable/meshos.ts +3 -0
- package/services/durable/workflow.ts +24 -22
- package/services/engine/index.ts +20 -5
- package/services/exporter/index.ts +147 -0
- package/services/hotmesh/index.ts +8 -1
- package/services/quorum/index.ts +37 -13
- package/services/router/index.ts +3 -0
- package/services/store/clients/redis.ts +1 -0
- package/services/store/index.ts +66 -14
- package/services/task/index.ts +4 -1
- package/services/worker/index.ts +6 -5
- package/types/activity.ts +6 -1
- package/types/exporter.ts +61 -0
- package/types/hotmesh.ts +1 -1
- package/types/index.ts +13 -1
- package/types/quorum.ts +1 -0
- package/types/task.ts +1 -1
package/build/modules/enums.d.ts
CHANGED
|
@@ -15,6 +15,8 @@ export declare const HMSH_CODE_DURABLE_MAXED = 597;
|
|
|
15
15
|
export declare const HMSH_CODE_DURABLE_FATAL = 598;
|
|
16
16
|
export declare const HMSH_CODE_DURABLE_RETRYABLE = 599;
|
|
17
17
|
export declare const HMSH_STATUS_UNKNOWN = "unknown";
|
|
18
|
+
export declare const HMSH_QUORUM_DELAY_MS = 250;
|
|
19
|
+
export declare const HMSH_ACTIVATION_MAX_RETRY = 3;
|
|
18
20
|
export declare const HMSH_OTT_WAIT_TIME: number;
|
|
19
21
|
export declare const HMSH_EXPIRE_JOB_SECONDS: number;
|
|
20
22
|
export declare const HMSH_MAX_RETRIES: number;
|
package/build/modules/enums.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
|
|
3
|
+
exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAITFOR = exports.HMSH_CODE_DURABLE_INCOMPLETE = exports.HMSH_CODE_DURABLE_SLEEPFOR = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
|
|
4
4
|
// HOTMESH SYSTEM
|
|
5
5
|
exports.HMSH_LOGLEVEL = process.env.HMSH_LOGLEVEL || 'info';
|
|
6
6
|
// STATUS CODES AND MESSAGES
|
|
@@ -19,6 +19,9 @@ exports.HMSH_CODE_DURABLE_MAXED = 597;
|
|
|
19
19
|
exports.HMSH_CODE_DURABLE_FATAL = 598;
|
|
20
20
|
exports.HMSH_CODE_DURABLE_RETRYABLE = 599;
|
|
21
21
|
exports.HMSH_STATUS_UNKNOWN = 'unknown';
|
|
22
|
+
// QUORUM
|
|
23
|
+
exports.HMSH_QUORUM_DELAY_MS = 250;
|
|
24
|
+
exports.HMSH_ACTIVATION_MAX_RETRY = 3;
|
|
22
25
|
// ENGINE
|
|
23
26
|
exports.HMSH_OTT_WAIT_TIME = parseInt(process.env.HMSH_OTT_WAIT_TIME, 10) || 1000;
|
|
24
27
|
exports.HMSH_EXPIRE_JOB_SECONDS = parseInt(process.env.HMSH_EXPIRE_JOB_SECONDS, 10) || 1;
|
package/build/modules/utils.js
CHANGED
package/build/package.json
CHANGED
|
@@ -158,11 +158,17 @@ class Trigger extends activity_1.Activity {
|
|
|
158
158
|
async registerJobDependency(multi) {
|
|
159
159
|
const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
|
|
160
160
|
let resolvedDepKey = depKey ? pipe_1.Pipe.resolve(depKey, this.context) : '';
|
|
161
|
+
const adjKey = this.config.stats?.adjacent;
|
|
162
|
+
let resolvedAdjKey = depKey ? pipe_1.Pipe.resolve(adjKey, this.context) : '';
|
|
161
163
|
if (!resolvedDepKey) {
|
|
162
164
|
resolvedDepKey = this.context.metadata.pj;
|
|
163
165
|
}
|
|
164
166
|
if (resolvedDepKey) {
|
|
165
|
-
|
|
167
|
+
const isParentOrigin = (resolvedDepKey === this.context.metadata.pj) || (resolvedDepKey === resolvedAdjKey);
|
|
168
|
+
await this.store.registerJobDependency(isParentOrigin ? 'expire-child' : 'expire', resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
|
|
169
|
+
}
|
|
170
|
+
if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
|
|
171
|
+
await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, multi);
|
|
166
172
|
}
|
|
167
173
|
}
|
|
168
174
|
async setStats(multi) {
|
|
@@ -3,8 +3,8 @@ import { HotMeshService as HotMesh } from '../hotmesh';
|
|
|
3
3
|
import { ClientConfig, Connection, HookOptions, WorkflowOptions, WorkflowSearchOptions } from '../../types/durable';
|
|
4
4
|
export declare class ClientService {
|
|
5
5
|
connection: Connection;
|
|
6
|
-
topics: string[];
|
|
7
6
|
options: WorkflowOptions;
|
|
7
|
+
static topics: string[];
|
|
8
8
|
static instances: Map<string, HotMesh | Promise<HotMesh>>;
|
|
9
9
|
constructor(config: ClientConfig);
|
|
10
10
|
getHotMeshClient: (workflowTopic: string, namespace?: string) => Promise<HotMesh>;
|
|
@@ -37,6 +37,7 @@ export declare class ClientService {
|
|
|
37
37
|
getHandle: (taskQueue: string, workflowName: string, workflowId: string, namespace?: string) => Promise<WorkflowHandleService>;
|
|
38
38
|
search: (taskQueue: string, workflowName: string, namespace: null | string, index: string, ...query: string[]) => Promise<string[]>;
|
|
39
39
|
};
|
|
40
|
+
verifyWorkflowActive(hotMesh: HotMesh, appId?: string, count?: number): Promise<boolean>;
|
|
40
41
|
activateWorkflow(hotMesh: HotMesh, appId?: string, version?: string): Promise<void>;
|
|
41
42
|
static shutdown(): Promise<void>;
|
|
42
43
|
}
|
|
@@ -8,16 +8,17 @@ const key_1 = require("../../modules/key");
|
|
|
8
8
|
const search_1 = require("./search");
|
|
9
9
|
const types_1 = require("../../types");
|
|
10
10
|
const enums_1 = require("../../modules/enums");
|
|
11
|
+
const utils_1 = require("../../modules/utils");
|
|
11
12
|
class ClientService {
|
|
12
13
|
constructor(config) {
|
|
13
|
-
this.topics = [];
|
|
14
14
|
this.getHotMeshClient = async (workflowTopic, namespace) => {
|
|
15
15
|
//use the cached instance
|
|
16
16
|
const instanceId = 'SINGLETON';
|
|
17
17
|
if (ClientService.instances.has(instanceId)) {
|
|
18
18
|
const hotMeshClient = await ClientService.instances.get(instanceId);
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
await this.verifyWorkflowActive(hotMeshClient, namespace ?? factory_1.APP_ID);
|
|
20
|
+
if (!ClientService.topics.includes(workflowTopic)) {
|
|
21
|
+
ClientService.topics.push(workflowTopic);
|
|
21
22
|
await this.createStream(hotMeshClient, workflowTopic, namespace);
|
|
22
23
|
}
|
|
23
24
|
return hotMeshClient;
|
|
@@ -174,6 +175,18 @@ class ClientService {
|
|
|
174
175
|
};
|
|
175
176
|
this.connection = config.connection;
|
|
176
177
|
}
|
|
178
|
+
async verifyWorkflowActive(hotMesh, appId = factory_1.APP_ID, count = 0) {
|
|
179
|
+
const app = await hotMesh.engine.store.getApp(appId);
|
|
180
|
+
const appVersion = app?.version;
|
|
181
|
+
if (isNaN(appVersion)) {
|
|
182
|
+
if (count > 10) {
|
|
183
|
+
throw new Error('Workflow failed to activate');
|
|
184
|
+
}
|
|
185
|
+
await (0, utils_1.sleepFor)(enums_1.HMSH_QUORUM_DELAY_MS * 2);
|
|
186
|
+
return await this.verifyWorkflowActive(hotMesh, appId, count + 1);
|
|
187
|
+
}
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
177
190
|
async activateWorkflow(hotMesh, appId = factory_1.APP_ID, version = factory_1.APP_VERSION) {
|
|
178
191
|
const app = await hotMesh.engine.store.getApp(appId);
|
|
179
192
|
const appVersion = app?.version;
|
|
@@ -203,5 +216,6 @@ class ClientService {
|
|
|
203
216
|
}
|
|
204
217
|
}
|
|
205
218
|
}
|
|
219
|
+
ClientService.topics = [];
|
|
206
220
|
ClientService.instances = new Map();
|
|
207
221
|
exports.ClientService = ClientService;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { ILogger } from '../logger';
|
|
2
|
+
import { StoreService } from '../store';
|
|
3
|
+
import { StringStringType, Symbols } from "../../types";
|
|
4
|
+
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
5
|
+
import { ActivityAction, DependencyExport, ExportItem, ExportOptions, JobAction, JobActionExport, DurableJobExport, JobTimeline } from '../../types/exporter';
|
|
6
|
+
import { SerializerService } from '../serializer';
|
|
7
|
+
/**
|
|
8
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
9
|
+
* Splits, Inflates, and Sorts the job data for use in durable contexts
|
|
10
|
+
*/
|
|
11
|
+
declare class ExporterService {
|
|
12
|
+
appId: string;
|
|
13
|
+
logger: ILogger;
|
|
14
|
+
serializer: SerializerService;
|
|
15
|
+
store: StoreService<RedisClient, RedisMulti>;
|
|
16
|
+
symbols: Promise<Symbols> | Symbols;
|
|
17
|
+
/**
|
|
18
|
+
* Friendly names for the activity ids
|
|
19
|
+
*/
|
|
20
|
+
activitySymbols: Symbols;
|
|
21
|
+
transitions: {
|
|
22
|
+
trigger: string[];
|
|
23
|
+
pivot: string[];
|
|
24
|
+
worker: string[];
|
|
25
|
+
sleeper: string[];
|
|
26
|
+
awaiter: string[];
|
|
27
|
+
retryer: string[];
|
|
28
|
+
hook: string[];
|
|
29
|
+
hook_pivot: string[];
|
|
30
|
+
hook_worker: string[];
|
|
31
|
+
hook_sleeper: string[];
|
|
32
|
+
hook_awaiter: string[];
|
|
33
|
+
hook_retryer: string[];
|
|
34
|
+
};
|
|
35
|
+
cycles: {
|
|
36
|
+
sleep_cycler: string[];
|
|
37
|
+
await_cycler: string[];
|
|
38
|
+
retry_cycler: string[];
|
|
39
|
+
hook_sleep_cycler: string[];
|
|
40
|
+
hook_await_cycler: string[];
|
|
41
|
+
hook_retry_cycler: string[];
|
|
42
|
+
};
|
|
43
|
+
constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
|
|
44
|
+
/**
|
|
45
|
+
* Convert the job hash and dependency list into a DurableJobExport object.
|
|
46
|
+
* This object contains various facets that describe the interaction
|
|
47
|
+
* in terms relevant to narrative storytelling.
|
|
48
|
+
*/
|
|
49
|
+
export(jobId: string, options?: ExportOptions): Promise<DurableJobExport>;
|
|
50
|
+
/**
|
|
51
|
+
* Interleave actions into the replay timeline to create
|
|
52
|
+
* a time-ordered timeline of the entire interaction, beginning
|
|
53
|
+
* with the entry trigger and concluding with the scrubber
|
|
54
|
+
* activity. Using the returned timeline, it is possible to
|
|
55
|
+
* create an animated narrative of the job, highlighting
|
|
56
|
+
* activities in the graph according to the timeline's
|
|
57
|
+
* activity-created (/ac) and activity-updated (/au) entries.
|
|
58
|
+
*/
|
|
59
|
+
createTimeline(replay: ExportItem[], actions: JobActionExport): JobTimeline[];
|
|
60
|
+
/**
|
|
61
|
+
* Interleave actions into the 'worker' and 'hook_worker'
|
|
62
|
+
* activities (between their /ac and /au entries)
|
|
63
|
+
*/
|
|
64
|
+
interleaveActions(target: JobAction, actions: ActivityAction[]): void;
|
|
65
|
+
isPausingAction(actionType: string): boolean;
|
|
66
|
+
isMainEntry(key: string): boolean;
|
|
67
|
+
isHookEntry(key: string): boolean;
|
|
68
|
+
/**
|
|
69
|
+
* Inflates the key from Redis, 3-character symbol
|
|
70
|
+
* into a human-readable JSON path, reflecting the
|
|
71
|
+
* tree-like structure of the unidimensional Hash
|
|
72
|
+
*/
|
|
73
|
+
inflateKey(key: string): string;
|
|
74
|
+
/**
|
|
75
|
+
* Inflates the dependency data from Redis into a DurableJobExport object by
|
|
76
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
77
|
+
* into a story
|
|
78
|
+
* @param data - the dependency data from Redis
|
|
79
|
+
* @returns - the organized dependency data
|
|
80
|
+
*/
|
|
81
|
+
inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
|
|
82
|
+
/**
|
|
83
|
+
* Adds historical actions (proxyActivity, executeChild)
|
|
84
|
+
* using the `dependency list` to determine
|
|
85
|
+
* after-the-fact what happened within the 'black-box'
|
|
86
|
+
* worker function. This is necessary to interleave the
|
|
87
|
+
* actions into the replay timeline, given that it isn't
|
|
88
|
+
* really possible to know the inner-workings of the user's
|
|
89
|
+
* function
|
|
90
|
+
*
|
|
91
|
+
*/
|
|
92
|
+
seedActions(type: 'flow' | 'hook' | 'other', action: string, topic: string, dep: string, prefix: string, dimensionKey: string, actions: JobActionExport, jobId: string): void;
|
|
93
|
+
/**
|
|
94
|
+
* Inflates the job data from Redis into a DurableJobExport object
|
|
95
|
+
* @param jobHash - the job data from Redis
|
|
96
|
+
* @param dependencyList - the list of dependencies for the job
|
|
97
|
+
* @returns - the inflated job data
|
|
98
|
+
*/
|
|
99
|
+
inflate(jobHash: StringStringType, dependencyList: string[]): DurableJobExport;
|
|
100
|
+
inflateProcess(match: RegExpMatchArray, value: string, replay: ExportItem[]): void;
|
|
101
|
+
inflateActions(key: string, value: string, actions: JobActionExport): void;
|
|
102
|
+
reverseSort(aKey: ExportItem, bKey: ExportItem): 1 | -1 | 0;
|
|
103
|
+
dateSort(aKey: ExportItem, bKey: ExportItem): 1 | -1 | 0;
|
|
104
|
+
}
|
|
105
|
+
export { ExporterService };
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExporterService = void 0;
|
|
4
|
+
const serializer_1 = require("../serializer");
|
|
5
|
+
const utils_1 = require("../../modules/utils");
|
|
6
|
+
/**
|
|
7
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
8
|
+
* Splits, Inflates, and Sorts the job data for use in durable contexts
|
|
9
|
+
*/
|
|
10
|
+
class ExporterService {
|
|
11
|
+
constructor(appId, store, logger) {
|
|
12
|
+
/**
|
|
13
|
+
* Friendly names for the activity ids
|
|
14
|
+
*/
|
|
15
|
+
this.activitySymbols = {
|
|
16
|
+
t1: 'trigger',
|
|
17
|
+
a1: 'pivot',
|
|
18
|
+
w1: 'worker',
|
|
19
|
+
a592: 'sleeper',
|
|
20
|
+
a594: 'awaiter',
|
|
21
|
+
a599: 'retryer',
|
|
22
|
+
c592: 'sleep_cycler',
|
|
23
|
+
c594: 'await_cycler',
|
|
24
|
+
c599: 'retry_cycler',
|
|
25
|
+
s5: 'scrubber',
|
|
26
|
+
sig: 'hook',
|
|
27
|
+
siga1: 'hook_pivot',
|
|
28
|
+
sigw1: 'hook_worker',
|
|
29
|
+
siga592: 'hook_sleeper',
|
|
30
|
+
siga594: 'hook_awaiter',
|
|
31
|
+
siga599: 'hook_retryer',
|
|
32
|
+
sigc592: 'hook_sleep_cycler',
|
|
33
|
+
sigc594: 'hook_await_cycler',
|
|
34
|
+
sigc599: 'hook_retry_cycler',
|
|
35
|
+
};
|
|
36
|
+
//adjacent transitions
|
|
37
|
+
this.transitions = {
|
|
38
|
+
trigger: ['pivot', 'hook'],
|
|
39
|
+
pivot: ['worker'],
|
|
40
|
+
worker: ['sleeper', 'awaiter', 'retryer', 'scrubber'],
|
|
41
|
+
sleeper: ['sleep_cycler'],
|
|
42
|
+
awaiter: ['await_cycler'],
|
|
43
|
+
retryer: ['retry_cycler'],
|
|
44
|
+
hook: ['hook_pivot'],
|
|
45
|
+
hook_pivot: ['hook_worker'],
|
|
46
|
+
hook_worker: ['hook_sleeper', 'hook_awaiter', 'hook_retryer'],
|
|
47
|
+
hook_sleeper: ['hook_sleep_cycler'],
|
|
48
|
+
hook_awaiter: ['hook_await_cycler'],
|
|
49
|
+
hook_retryer: ['hook_retry_cycler'],
|
|
50
|
+
};
|
|
51
|
+
//goto transitions
|
|
52
|
+
this.cycles = {
|
|
53
|
+
sleep_cycler: ['pivot'],
|
|
54
|
+
await_cycler: ['pivot'],
|
|
55
|
+
retry_cycler: ['pivot'],
|
|
56
|
+
hook_sleep_cycler: ['hook_pivot'],
|
|
57
|
+
hook_await_cycler: ['hook_pivot'],
|
|
58
|
+
hook_retry_cycler: ['hook_pivot'],
|
|
59
|
+
};
|
|
60
|
+
this.appId = appId;
|
|
61
|
+
this.logger = logger;
|
|
62
|
+
this.store = store;
|
|
63
|
+
this.serializer = new serializer_1.SerializerService();
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Convert the job hash and dependency list into a DurableJobExport object.
|
|
67
|
+
* This object contains various facets that describe the interaction
|
|
68
|
+
* in terms relevant to narrative storytelling.
|
|
69
|
+
*/
|
|
70
|
+
async export(jobId, options = {}) {
|
|
71
|
+
if (!this.symbols) {
|
|
72
|
+
this.symbols = this.store.getAllSymbols();
|
|
73
|
+
this.symbols = await this.symbols;
|
|
74
|
+
}
|
|
75
|
+
const depData = await this.store.getDependencies(jobId);
|
|
76
|
+
const jobData = await this.store.getRaw(jobId);
|
|
77
|
+
const jobExport = this.inflate(jobData, depData);
|
|
78
|
+
return jobExport;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Interleave actions into the replay timeline to create
|
|
82
|
+
* a time-ordered timeline of the entire interaction, beginning
|
|
83
|
+
* with the entry trigger and concluding with the scrubber
|
|
84
|
+
* activity. Using the returned timeline, it is possible to
|
|
85
|
+
* create an animated narrative of the job, highlighting
|
|
86
|
+
* activities in the graph according to the timeline's
|
|
87
|
+
* activity-created (/ac) and activity-updated (/au) entries.
|
|
88
|
+
*/
|
|
89
|
+
createTimeline(replay, actions) {
|
|
90
|
+
const timeline = [];
|
|
91
|
+
replay.forEach((item) => {
|
|
92
|
+
const dimensions = item[0];
|
|
93
|
+
const parts = dimensions.split('/');
|
|
94
|
+
const activityName = item[1].split('/')[0];
|
|
95
|
+
const duplex = item[1].endsWith('/ac') ? 'entry' : 'exit';
|
|
96
|
+
const timestamp = item[2];
|
|
97
|
+
const event = {
|
|
98
|
+
activity: activityName,
|
|
99
|
+
duplex: duplex,
|
|
100
|
+
dimension: dimensions,
|
|
101
|
+
timestamp,
|
|
102
|
+
};
|
|
103
|
+
timeline.push(event);
|
|
104
|
+
if (this.isMainEntry(item[1])) {
|
|
105
|
+
event.actions = [];
|
|
106
|
+
this.interleaveActions(actions.main, event.actions);
|
|
107
|
+
}
|
|
108
|
+
else if (this.isHookEntry(item[1])) {
|
|
109
|
+
const hookDimension = `/${parts[1]}/${parts[2]}`;
|
|
110
|
+
const hookActions = actions.hooks[hookDimension];
|
|
111
|
+
event.actions = [];
|
|
112
|
+
this.interleaveActions(hookActions, event.actions);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
return timeline;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Interleave actions into the 'worker' and 'hook_worker'
|
|
119
|
+
* activities (between their /ac and /au entries)
|
|
120
|
+
*/
|
|
121
|
+
interleaveActions(target, actions) {
|
|
122
|
+
if (target) {
|
|
123
|
+
for (let i = target.cursor + 1; i < target.items.length; i++) {
|
|
124
|
+
const [_, actionType, jobOrIndex] = target.items[i];
|
|
125
|
+
actions.push({ action: actionType, target: jobOrIndex });
|
|
126
|
+
target.cursor = i;
|
|
127
|
+
if (this.isPausingAction(actionType)) {
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
isPausingAction(actionType) {
|
|
134
|
+
return actionType === 'sleep' || actionType === 'waitForSignal';
|
|
135
|
+
}
|
|
136
|
+
isMainEntry(key) {
|
|
137
|
+
return key.startsWith('worker/') && key.endsWith('/ac');
|
|
138
|
+
}
|
|
139
|
+
isHookEntry(key) {
|
|
140
|
+
return key.startsWith('hook_worker/') && key.endsWith('/ac');
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Inflates the key from Redis, 3-character symbol
|
|
144
|
+
* into a human-readable JSON path, reflecting the
|
|
145
|
+
* tree-like structure of the unidimensional Hash
|
|
146
|
+
*/
|
|
147
|
+
inflateKey(key) {
|
|
148
|
+
if (key in this.symbols) {
|
|
149
|
+
const path = this.symbols[key];
|
|
150
|
+
const parts = path.split('/');
|
|
151
|
+
if (parts[0] in this.activitySymbols) {
|
|
152
|
+
parts[0] = this.activitySymbols[parts[0]];
|
|
153
|
+
}
|
|
154
|
+
return parts.join('/');
|
|
155
|
+
}
|
|
156
|
+
return key;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Inflates the dependency data from Redis into a DurableJobExport object by
|
|
160
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
161
|
+
* into a story
|
|
162
|
+
* @param data - the dependency data from Redis
|
|
163
|
+
* @returns - the organized dependency data
|
|
164
|
+
*/
|
|
165
|
+
inflateDependencyData(data, actions) {
|
|
166
|
+
//console.log('dependency data>', data);
|
|
167
|
+
const hookReg = /([0-9,]+)-(\d+)$/;
|
|
168
|
+
const flowReg = /-(\d+)$/;
|
|
169
|
+
return data.map((dependency, index) => {
|
|
170
|
+
const [action, topic, gid, ...jid] = dependency.split('::');
|
|
171
|
+
const jobId = jid.join('::');
|
|
172
|
+
const match = jobId.match(hookReg);
|
|
173
|
+
let prefix;
|
|
174
|
+
let type;
|
|
175
|
+
let dimensionKey = '';
|
|
176
|
+
if (match) {
|
|
177
|
+
//hook-originating dependency
|
|
178
|
+
const [_, dimension, counter] = match;
|
|
179
|
+
dimensionKey = dimension.split(',').join('/');
|
|
180
|
+
prefix = `${dimensionKey}[${counter}]`;
|
|
181
|
+
type = 'hook';
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
const match = jobId.match(flowReg);
|
|
185
|
+
if (match) {
|
|
186
|
+
//main workflow-originating dependency
|
|
187
|
+
const [_, counter] = match;
|
|
188
|
+
prefix = `[${counter}]`;
|
|
189
|
+
type = 'flow';
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
//'other' types like signal cleanup
|
|
193
|
+
prefix = '/';
|
|
194
|
+
type = 'other';
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
this.seedActions(type, action, topic, dependency, prefix, dimensionKey, actions, jobId);
|
|
198
|
+
return {
|
|
199
|
+
type: action,
|
|
200
|
+
topic,
|
|
201
|
+
gid,
|
|
202
|
+
jid: jobId,
|
|
203
|
+
};
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Adds historical actions (proxyActivity, executeChild)
|
|
208
|
+
* using the `dependency list` to determine
|
|
209
|
+
* after-the-fact what happened within the 'black-box'
|
|
210
|
+
* worker function. This is necessary to interleave the
|
|
211
|
+
* actions into the replay timeline, given that it isn't
|
|
212
|
+
* really possible to know the inner-workings of the user's
|
|
213
|
+
* function
|
|
214
|
+
*
|
|
215
|
+
*/
|
|
216
|
+
seedActions(type, action, topic, dep, prefix, dimensionKey, actions, jobId) {
|
|
217
|
+
if (type !== 'other' && action === 'expire-child') {
|
|
218
|
+
let depType;
|
|
219
|
+
if (topic == `${this.appId}.activity.execute`) {
|
|
220
|
+
depType = 'proxyActivity';
|
|
221
|
+
}
|
|
222
|
+
else if (topic == `${this.appId}.execute`) {
|
|
223
|
+
depType = 'executeChild';
|
|
224
|
+
}
|
|
225
|
+
else if (topic == `${this.appId}.wfsc.execute`) {
|
|
226
|
+
depType = 'waitForSignal';
|
|
227
|
+
}
|
|
228
|
+
if (depType) {
|
|
229
|
+
if (type === 'flow') {
|
|
230
|
+
actions.main.items.push([prefix, depType, jobId]);
|
|
231
|
+
}
|
|
232
|
+
else if (type === 'hook') {
|
|
233
|
+
if (!actions.hooks[dimensionKey]) {
|
|
234
|
+
actions.hooks[dimensionKey] = {
|
|
235
|
+
cursor: -1,
|
|
236
|
+
items: [],
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
actions.hooks[dimensionKey].items.push([prefix, depType, jobId]);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Inflates the job data from Redis into a DurableJobExport object
|
|
246
|
+
* @param jobHash - the job data from Redis
|
|
247
|
+
* @param dependencyList - the list of dependencies for the job
|
|
248
|
+
* @returns - the inflated job data
|
|
249
|
+
*/
|
|
250
|
+
inflate(jobHash, dependencyList) {
|
|
251
|
+
//the list of actions taken in the workflow and hook functions
|
|
252
|
+
const actions = {
|
|
253
|
+
hooks: {},
|
|
254
|
+
main: { cursor: -1, items: [] },
|
|
255
|
+
};
|
|
256
|
+
const dependencies = this.inflateDependencyData(dependencyList, actions);
|
|
257
|
+
const state = {};
|
|
258
|
+
const data = {};
|
|
259
|
+
const other = [];
|
|
260
|
+
const replay = [];
|
|
261
|
+
const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
|
|
262
|
+
Object.entries(jobHash).forEach(([key, value]) => {
|
|
263
|
+
const match = key.match(regex);
|
|
264
|
+
if (match) {
|
|
265
|
+
//activity process state
|
|
266
|
+
this.inflateProcess(match, value, replay);
|
|
267
|
+
}
|
|
268
|
+
else if (key.length === 3) {
|
|
269
|
+
//job state
|
|
270
|
+
state[this.inflateKey(key)] = this.serializer.fromString(value);
|
|
271
|
+
}
|
|
272
|
+
else if (key.startsWith('_')) {
|
|
273
|
+
//job data
|
|
274
|
+
data[key.substring(1)] = value;
|
|
275
|
+
}
|
|
276
|
+
else if (key.startsWith('-')) {
|
|
277
|
+
//actions with side effect (replayable)
|
|
278
|
+
this.inflateActions(key, value, actions);
|
|
279
|
+
}
|
|
280
|
+
else {
|
|
281
|
+
//collator guids, etc
|
|
282
|
+
other.push([null, key, value]);
|
|
283
|
+
}
|
|
284
|
+
});
|
|
285
|
+
replay.sort(this.dateSort);
|
|
286
|
+
actions.main.items.sort(this.reverseSort);
|
|
287
|
+
Object.entries(actions.hooks).forEach(([key, value]) => {
|
|
288
|
+
value.items.sort(this.reverseSort);
|
|
289
|
+
});
|
|
290
|
+
return {
|
|
291
|
+
data: (0, utils_1.restoreHierarchy)(data),
|
|
292
|
+
dependencies,
|
|
293
|
+
state: Object.entries((0, utils_1.restoreHierarchy)(state))[0][1],
|
|
294
|
+
status: jobHash[':'],
|
|
295
|
+
timeline: this.createTimeline(replay, actions),
|
|
296
|
+
transitions: { ...this.transitions },
|
|
297
|
+
cycles: { ...this.cycles },
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
inflateProcess(match, value, replay) {
|
|
301
|
+
const [_, letters, numbers] = match;
|
|
302
|
+
const path = this.inflateKey(letters);
|
|
303
|
+
if (path.endsWith('/output/metadata/ac') ||
|
|
304
|
+
path.endsWith('/output/metadata/au')) {
|
|
305
|
+
const dimensions = `/${numbers.replace(/,/g, '/')}`;
|
|
306
|
+
const resolved = this.serializer.fromString(value);
|
|
307
|
+
replay.push([
|
|
308
|
+
dimensions,
|
|
309
|
+
path,
|
|
310
|
+
resolved,
|
|
311
|
+
]);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
inflateActions(key, value, actions) {
|
|
315
|
+
let [_, dimensionalType, counter, subcounter] = key.split('-');
|
|
316
|
+
if (subcounter) {
|
|
317
|
+
counter = `${counter}.${subcounter}`;
|
|
318
|
+
}
|
|
319
|
+
const [type, ...dimensions] = dimensionalType.split(',');
|
|
320
|
+
let dimensionKey = '';
|
|
321
|
+
let isHook = false;
|
|
322
|
+
if (dimensions.length > 0) {
|
|
323
|
+
dimensionKey = `/${dimensions.join('/')}`;
|
|
324
|
+
isHook = true;
|
|
325
|
+
}
|
|
326
|
+
let targetList;
|
|
327
|
+
if (isHook) {
|
|
328
|
+
if (!actions.hooks[dimensionKey]) {
|
|
329
|
+
actions.hooks[dimensionKey] = {
|
|
330
|
+
cursor: -1,
|
|
331
|
+
items: [],
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
targetList = actions.hooks[dimensionKey].items;
|
|
335
|
+
}
|
|
336
|
+
else {
|
|
337
|
+
targetList = actions.main.items;
|
|
338
|
+
}
|
|
339
|
+
targetList.push([
|
|
340
|
+
`${dimensionKey}[${counter}]`,
|
|
341
|
+
type,
|
|
342
|
+
value,
|
|
343
|
+
]);
|
|
344
|
+
}
|
|
345
|
+
reverseSort(aKey, bKey) {
|
|
346
|
+
if (aKey[0] > bKey[0]) {
|
|
347
|
+
return 1;
|
|
348
|
+
}
|
|
349
|
+
else if (aKey[0] < bKey[0]) {
|
|
350
|
+
return -1;
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
if (aKey[1] > bKey[1]) {
|
|
354
|
+
return 1;
|
|
355
|
+
}
|
|
356
|
+
else if (aKey[1] < bKey[1]) {
|
|
357
|
+
return -1;
|
|
358
|
+
}
|
|
359
|
+
return 0;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
dateSort(aKey, bKey) {
|
|
363
|
+
if (aKey[2] > bKey[2]) {
|
|
364
|
+
return 1;
|
|
365
|
+
}
|
|
366
|
+
else if (aKey[2] < bKey[2]) {
|
|
367
|
+
return -1;
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
return 0;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
exports.ExporterService = ExporterService;
|