@hotmeshio/hotmesh 0.0.43 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/package.json +1 -1
- package/build/services/activities/trigger.js +7 -1
- package/build/services/durable/client.js +7 -8
- package/build/services/durable/exporter.d.ts +105 -0
- package/build/services/durable/exporter.js +374 -0
- package/build/services/durable/factory.js +6 -63
- package/build/services/durable/handle.d.ts +4 -0
- package/build/services/durable/handle.js +5 -0
- package/build/services/durable/workflow.js +24 -21
- package/build/services/engine/index.d.ts +6 -1
- package/build/services/engine/index.js +9 -2
- package/build/services/exporter/index.d.ts +46 -0
- package/build/services/exporter/index.js +126 -0
- package/build/services/hotmesh/index.d.ts +4 -1
- package/build/services/hotmesh/index.js +6 -0
- package/build/services/quorum/index.js +4 -1
- package/build/services/router/index.d.ts +3 -0
- package/build/services/router/index.js +3 -0
- package/build/services/store/index.d.ts +5 -2
- package/build/services/store/index.js +54 -6
- package/build/services/task/index.js +5 -1
- package/build/services/worker/index.js +6 -4
- package/build/types/activity.d.ts +6 -1
- package/build/types/exporter.d.ts +51 -0
- package/build/types/exporter.js +8 -0
- package/build/types/index.d.ts +1 -0
- package/build/types/quorum.d.ts +2 -0
- package/build/types/task.d.ts +1 -1
- package/package.json +1 -1
- package/services/activities/trigger.ts +14 -0
- package/services/durable/client.ts +7 -8
- package/services/durable/exporter.ts +408 -0
- package/services/durable/factory.ts +6 -63
- package/services/durable/handle.ts +12 -0
- package/services/durable/workflow.ts +24 -22
- package/services/engine/index.ts +20 -5
- package/services/exporter/index.ts +147 -0
- package/services/hotmesh/index.ts +8 -1
- package/services/quorum/index.ts +8 -2
- package/services/router/index.ts +4 -0
- package/services/store/index.ts +56 -7
- package/services/task/index.ts +4 -1
- package/services/worker/index.ts +7 -5
- package/types/activity.ts +6 -1
- package/types/exporter.ts +61 -0
- package/types/index.ts +13 -1
- package/types/quorum.ts +2 -0
- package/types/task.ts +1 -1
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { HMSH_CODE_INTERRUPT } from '../../modules/enums';
|
|
2
|
+
import { ExporterService } from './exporter';
|
|
2
3
|
import { HotMeshService as HotMesh } from '../hotmesh';
|
|
4
|
+
import { DurableJobExport } from '../../types/exporter';
|
|
3
5
|
import { JobInterruptOptions, JobOutput } from '../../types/job';
|
|
4
6
|
import { StreamError } from '../../types/stream';
|
|
5
7
|
|
|
6
8
|
export class WorkflowHandleService {
|
|
9
|
+
exporter: ExporterService
|
|
7
10
|
hotMesh: HotMesh;
|
|
8
11
|
workflowTopic: string;
|
|
9
12
|
workflowId: string;
|
|
@@ -12,6 +15,15 @@ export class WorkflowHandleService {
|
|
|
12
15
|
this.workflowTopic = workflowTopic;
|
|
13
16
|
this.workflowId = workflowId;
|
|
14
17
|
this.hotMesh = hotMesh;
|
|
18
|
+
this.exporter = new ExporterService(
|
|
19
|
+
this.hotMesh.appId,
|
|
20
|
+
this.hotMesh.engine.store,
|
|
21
|
+
this.hotMesh.engine.logger,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async export(): Promise<DurableJobExport> {
|
|
26
|
+
return this.exporter.export(this.workflowId);
|
|
15
27
|
}
|
|
16
28
|
|
|
17
29
|
/**
|
|
@@ -45,7 +45,7 @@ export class WorkflowService {
|
|
|
45
45
|
const entityOrEmptyString = options.entity ?? '';
|
|
46
46
|
//If the workflowId is not provided, it is generated from the entity and the workflow name
|
|
47
47
|
const childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
|
|
48
|
-
const parentWorkflowId =
|
|
48
|
+
const parentWorkflowId = workflowId;
|
|
49
49
|
|
|
50
50
|
const client = new Client({
|
|
51
51
|
connection: await Connection.connect(WorkerService.connection),
|
|
@@ -91,33 +91,35 @@ export class WorkflowService {
|
|
|
91
91
|
const workflowSpan = store.get('workflowSpan');
|
|
92
92
|
const COUNTER = store.get('counter');
|
|
93
93
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
94
|
+
const sessionId = `-start${workflowDimension}-${execIndex}-`;
|
|
94
95
|
//NOTE: this is the hash prefix; necessary for the search index to locate the entity
|
|
95
96
|
const entityOrEmptyString = options.entity ?? '';
|
|
96
97
|
//If the workflowId is not provided, it is generated from the entity and the workflow name
|
|
97
|
-
const
|
|
98
|
-
const parentWorkflowId = `${workflowId}-f`;
|
|
98
|
+
const parentWorkflowId = workflowId;
|
|
99
99
|
const workflowTopic = `${options.entity ?? options.taskQueue}-${options.entity ?? options.workflowName}`;
|
|
100
100
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
} catch (error) {
|
|
107
|
-
const client = new Client({
|
|
108
|
-
connection: await Connection.connect(WorkerService.connection),
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
await client.workflow.start({
|
|
112
|
-
...options,
|
|
113
|
-
namespace,
|
|
114
|
-
workflowId: childJobId,
|
|
115
|
-
parentWorkflowId,
|
|
116
|
-
workflowTrace,
|
|
117
|
-
workflowSpan,
|
|
118
|
-
});
|
|
101
|
+
const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
102
|
+
const keyParams = { appId: hotMeshClient.appId, jobId: workflowId }
|
|
103
|
+
const workflowGuid = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
|
|
104
|
+
let childJobId = await hotMeshClient.engine.store.exec('HGET', workflowGuid, sessionId) as string;
|
|
105
|
+
if (childJobId) {
|
|
119
106
|
return childJobId;
|
|
107
|
+
} else {
|
|
108
|
+
childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
|
|
120
109
|
}
|
|
110
|
+
const client = new Client({
|
|
111
|
+
connection: await Connection.connect(WorkerService.connection),
|
|
112
|
+
});
|
|
113
|
+
await client.workflow.start({
|
|
114
|
+
...options,
|
|
115
|
+
namespace,
|
|
116
|
+
workflowId: childJobId,
|
|
117
|
+
parentWorkflowId,
|
|
118
|
+
workflowTrace,
|
|
119
|
+
workflowSpan,
|
|
120
|
+
});
|
|
121
|
+
await hotMeshClient.engine.store.exec('HSET', workflowGuid, sessionId, childJobId);
|
|
122
|
+
return childJobId;
|
|
121
123
|
}
|
|
122
124
|
|
|
123
125
|
/**
|
|
@@ -472,7 +474,7 @@ export class WorkflowService {
|
|
|
472
474
|
arguments: Array.from(arguments),
|
|
473
475
|
//when the origin job is removed
|
|
474
476
|
originJobId: originJobId ?? workflowId,
|
|
475
|
-
parentWorkflowId:
|
|
477
|
+
parentWorkflowId: workflowId,
|
|
476
478
|
workflowId: activityJobId,
|
|
477
479
|
workflowTopic: activityTopic,
|
|
478
480
|
activityName,
|
package/services/engine/index.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { Signal } from '../activities/signal';
|
|
|
21
21
|
import { Worker } from '../activities/worker';
|
|
22
22
|
import { Trigger } from '../activities/trigger';
|
|
23
23
|
import { CompilerService } from '../compiler';
|
|
24
|
+
import { ExporterService } from '../exporter';
|
|
24
25
|
import { ILogger } from '../logger';
|
|
25
26
|
import { ReporterService } from '../reporter';
|
|
26
27
|
import { Router } from '../router';
|
|
@@ -78,12 +79,14 @@ import {
|
|
|
78
79
|
StreamRole,
|
|
79
80
|
StreamStatus } from '../../types/stream';
|
|
80
81
|
import { WorkListTaskType } from '../../types/task';
|
|
82
|
+
import { JobExport } from '../../types/exporter';
|
|
81
83
|
|
|
82
84
|
class EngineService {
|
|
83
85
|
namespace: string;
|
|
84
86
|
apps: HotMeshApps | null;
|
|
85
87
|
appId: string;
|
|
86
88
|
guid: string;
|
|
89
|
+
exporter: ExporterService | null;
|
|
87
90
|
router: Router | null;
|
|
88
91
|
store: StoreService<RedisClient, RedisMulti> | null;
|
|
89
92
|
stream: StreamService<RedisClient, RedisMulti> | null;
|
|
@@ -109,8 +112,8 @@ class EngineService {
|
|
|
109
112
|
await instance.initStoreChannel(config.engine.store);
|
|
110
113
|
await instance.initSubChannel(config.engine.sub);
|
|
111
114
|
await instance.initStreamChannel(config.engine.stream);
|
|
112
|
-
instance.router = instance.initRouter(config);
|
|
113
115
|
|
|
116
|
+
instance.router = instance.initRouter(config);
|
|
114
117
|
instance.router.consumeMessages(
|
|
115
118
|
instance.stream.mintKey(
|
|
116
119
|
KeyType.STREAMS,
|
|
@@ -121,9 +124,15 @@ class EngineService {
|
|
|
121
124
|
instance.processStreamMessage.bind(instance)
|
|
122
125
|
);
|
|
123
126
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
+
instance.taskService = new TaskService(
|
|
128
|
+
instance.store,
|
|
129
|
+
logger
|
|
130
|
+
);
|
|
131
|
+
instance.exporter = new ExporterService(
|
|
132
|
+
instance.appId,
|
|
133
|
+
instance.store,
|
|
134
|
+
logger,
|
|
135
|
+
);
|
|
127
136
|
return instance;
|
|
128
137
|
}
|
|
129
138
|
}
|
|
@@ -690,9 +699,15 @@ class EngineService {
|
|
|
690
699
|
|
|
691
700
|
|
|
692
701
|
// ****** GET JOB STATE/COLLATION STATUS BY ID *********
|
|
702
|
+
async export(jobId: string): Promise<JobExport> {
|
|
703
|
+
return await this.exporter.export(jobId);
|
|
704
|
+
}
|
|
705
|
+
async getRaw(jobId: string): Promise<StringStringType> {
|
|
706
|
+
return await this.store.getRaw(jobId);
|
|
707
|
+
}
|
|
693
708
|
async getStatus(jobId: string): Promise<JobStatus> {
|
|
694
709
|
const { id: appId } = await this.getVID();
|
|
695
|
-
return this.store.getStatus(jobId, appId);
|
|
710
|
+
return await this.store.getStatus(jobId, appId);
|
|
696
711
|
}
|
|
697
712
|
//todo: add 'options' parameter;
|
|
698
713
|
// (e.g, if {dimensions:true}, use hscan to deliver
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { ILogger } from '../logger';
|
|
2
|
+
import { StoreService } from '../store';
|
|
3
|
+
import {
|
|
4
|
+
StringAnyType,
|
|
5
|
+
StringStringType,
|
|
6
|
+
Symbols } from "../../types/serializer";
|
|
7
|
+
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
8
|
+
import {
|
|
9
|
+
DependencyExport,
|
|
10
|
+
ExportOptions,
|
|
11
|
+
JobActionExport,
|
|
12
|
+
JobExport } from '../../types/exporter';
|
|
13
|
+
import { SerializerService } from '../serializer';
|
|
14
|
+
import { restoreHierarchy } from '../../modules/utils';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
18
|
+
* Expands process data and includes dependency list
|
|
19
|
+
*/
|
|
20
|
+
class ExporterService {
|
|
21
|
+
appId: string;
|
|
22
|
+
logger: ILogger;
|
|
23
|
+
serializer: SerializerService
|
|
24
|
+
store: StoreService<RedisClient, RedisMulti>;
|
|
25
|
+
symbols: Promise<Symbols> | Symbols;
|
|
26
|
+
|
|
27
|
+
constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
|
|
28
|
+
this.appId = appId;
|
|
29
|
+
this.logger = logger;
|
|
30
|
+
this.store = store;
|
|
31
|
+
this.serializer = new SerializerService();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Convert the job hash and dependency list into a JobExport object.
|
|
36
|
+
* This object contains various facets that describe the interaction
|
|
37
|
+
* in terms relevant to narrative storytelling.
|
|
38
|
+
*/
|
|
39
|
+
async export(jobId: string, options: ExportOptions = {}): Promise<JobExport> {
|
|
40
|
+
if (!this.symbols) {
|
|
41
|
+
this.symbols = this.store.getAllSymbols();
|
|
42
|
+
this.symbols = await this.symbols;
|
|
43
|
+
}
|
|
44
|
+
const depData = await this.store.getDependencies(jobId);
|
|
45
|
+
const jobData = await this.store.getRaw(jobId);
|
|
46
|
+
const jobExport = this.inflate(jobData, depData);
|
|
47
|
+
return jobExport;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Inflates the key from Redis, 3-character symbol
|
|
52
|
+
* into a human-readable JSON path, reflecting the
|
|
53
|
+
* tree-like structure of the unidimensional Hash
|
|
54
|
+
*/
|
|
55
|
+
inflateKey(key: string): string {
|
|
56
|
+
return (key in this.symbols) ? this.symbols[key] : key;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Inflates the job data from Redis into a JobExport object
|
|
61
|
+
* @param jobHash - the job data from Redis
|
|
62
|
+
* @param dependencyList - the list of dependencies for the job
|
|
63
|
+
* @returns - the inflated job data
|
|
64
|
+
*/
|
|
65
|
+
inflate(jobHash: StringStringType, dependencyList: string[]): JobExport {
|
|
66
|
+
//the list of actions taken in the workflow and hook functions
|
|
67
|
+
const actions: JobActionExport = {
|
|
68
|
+
hooks: {},
|
|
69
|
+
main: {
|
|
70
|
+
cursor: -1,
|
|
71
|
+
items: []
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
const process: StringAnyType = {};
|
|
75
|
+
const dependencies = this.inflateDependencyData(dependencyList, actions);
|
|
76
|
+
const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
|
|
77
|
+
|
|
78
|
+
Object.entries(jobHash).forEach(([key, value]) => {
|
|
79
|
+
const match = key.match(regex);
|
|
80
|
+
if (match) {
|
|
81
|
+
//activity process state
|
|
82
|
+
const [_, letters, numbers] = match;
|
|
83
|
+
const path = this.inflateKey(letters);
|
|
84
|
+
const dimensions = `${numbers.replace(/,/g, '/')}`;
|
|
85
|
+
const resolved = this.serializer.fromString(value);
|
|
86
|
+
process[`${dimensions}/${path}`] = resolved;
|
|
87
|
+
} else if (key.length === 3) {
|
|
88
|
+
//job state
|
|
89
|
+
process[this.inflateKey(key)] = this.serializer.fromString(value);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
dependencies,
|
|
95
|
+
process: restoreHierarchy(process),
|
|
96
|
+
status: jobHash[':'],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Inflates the dependency data from Redis into a JobExport object by
|
|
102
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
103
|
+
* into a story
|
|
104
|
+
* @param data - the dependency data from Redis
|
|
105
|
+
* @returns - the organized dependency data
|
|
106
|
+
*/
|
|
107
|
+
inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[] {
|
|
108
|
+
const hookReg = /([0-9,]+)-(\d+)$/;
|
|
109
|
+
const flowReg = /-(\d+)$/;
|
|
110
|
+
return data.map((dependency, index: number): DependencyExport => {
|
|
111
|
+
const [action, topic, gid, ...jid] = dependency.split('::');
|
|
112
|
+
const jobId = jid.join('::');
|
|
113
|
+
const match = jobId.match(hookReg);
|
|
114
|
+
let prefix: string;
|
|
115
|
+
let type: 'hook' | 'flow' | 'other';
|
|
116
|
+
let dimensionKey: string = '';
|
|
117
|
+
|
|
118
|
+
if (match) {
|
|
119
|
+
//hook-originating dependency
|
|
120
|
+
const [_, dimension, counter] = match;
|
|
121
|
+
dimensionKey = dimension.split(',').join('/');
|
|
122
|
+
prefix = `${dimensionKey}[${counter}]`;
|
|
123
|
+
type = 'hook';
|
|
124
|
+
} else {
|
|
125
|
+
const match = jobId.match(flowReg);
|
|
126
|
+
if (match) {
|
|
127
|
+
//main workflow-originating dependency
|
|
128
|
+
const [_, counter] = match;
|
|
129
|
+
prefix = `[${counter}]`;
|
|
130
|
+
type = 'flow';
|
|
131
|
+
} else {
|
|
132
|
+
//'other' types like signal cleanup
|
|
133
|
+
prefix = '/';
|
|
134
|
+
type = 'other';
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
type: action,
|
|
139
|
+
topic,
|
|
140
|
+
gid,
|
|
141
|
+
jid: jobId,
|
|
142
|
+
} as unknown as DependencyExport;
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export { ExporterService };
|
|
@@ -24,7 +24,8 @@ import {
|
|
|
24
24
|
StatsResponse } from '../../types/stats';
|
|
25
25
|
import { ConnectorService } from '../connector';
|
|
26
26
|
import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
|
|
27
|
-
import { StringAnyType } from '../../types/serializer';
|
|
27
|
+
import { StringAnyType, StringStringType } from '../../types/serializer';
|
|
28
|
+
import { JobExport } from '../../types/exporter';
|
|
28
29
|
|
|
29
30
|
class HotMeshService {
|
|
30
31
|
namespace: string;
|
|
@@ -152,6 +153,12 @@ class HotMeshService {
|
|
|
152
153
|
}
|
|
153
154
|
|
|
154
155
|
// ************* REPORTER METHODS *************
|
|
156
|
+
async export(jobId: string): Promise<JobExport> {
|
|
157
|
+
return await this.engine?.export(jobId);
|
|
158
|
+
}
|
|
159
|
+
async getRaw(jobId: string): Promise<StringStringType> {
|
|
160
|
+
return await this.engine?.getRaw(jobId);
|
|
161
|
+
}
|
|
155
162
|
async getStats(topic: string, query: JobStatsInput): Promise<StatsResponse> {
|
|
156
163
|
return await this.engine?.getStats(topic, query);
|
|
157
164
|
}
|
package/services/quorum/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HMSH_ACTIVATION_MAX_RETRY, HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
|
|
2
|
-
import { identifyRedisType, sleepFor } from '../../modules/utils';
|
|
2
|
+
import { formatISODate, identifyRedisType, sleepFor } from '../../modules/utils';
|
|
3
3
|
import { CompilerService } from '../compiler';
|
|
4
4
|
import { EngineService } from '../engine';
|
|
5
5
|
import { ILogger } from '../logger';
|
|
@@ -141,11 +141,17 @@ class QuorumService {
|
|
|
141
141
|
async sayPong(appId: string, guid: string, originator: string, details = false) {
|
|
142
142
|
let profile: QuorumProfile;
|
|
143
143
|
if (details) {
|
|
144
|
+
const stream = this.engine.stream.mintKey(
|
|
145
|
+
KeyType.STREAMS,
|
|
146
|
+
{ appId: this.appId }
|
|
147
|
+
);
|
|
144
148
|
profile = {
|
|
145
149
|
engine_id: this.guid,
|
|
146
150
|
namespace: this.namespace,
|
|
147
151
|
app_id: this.appId,
|
|
148
|
-
stream
|
|
152
|
+
stream,
|
|
153
|
+
counts: this.engine.router.counts,
|
|
154
|
+
timestamp: formatISODate(new Date()),
|
|
149
155
|
};
|
|
150
156
|
}
|
|
151
157
|
this.store.publish(
|
package/services/router/index.ts
CHANGED
|
@@ -40,6 +40,7 @@ class Router {
|
|
|
40
40
|
logger: ILogger;
|
|
41
41
|
throttle = 0;
|
|
42
42
|
errorCount = 0;
|
|
43
|
+
counts: { [key: string]: number } = {};
|
|
43
44
|
currentTimerId: NodeJS.Timeout | null = null;
|
|
44
45
|
shouldConsume: boolean;
|
|
45
46
|
|
|
@@ -64,6 +65,9 @@ class Router {
|
|
|
64
65
|
}
|
|
65
66
|
|
|
66
67
|
async publishMessage(topic: string, streamData: StreamData|StreamDataResponse, multi?: RedisMulti): Promise<string | RedisMulti> {
|
|
68
|
+
const code = streamData?.code || '200';
|
|
69
|
+
this.counts[code] = (this.counts[code] || 0) + 1;
|
|
70
|
+
|
|
67
71
|
const stream = this.store.mintKey(KeyType.STREAMS, { appId: this.store.appId, topic });
|
|
68
72
|
return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
|
|
69
73
|
}
|
package/services/store/index.ts
CHANGED
|
@@ -241,6 +241,36 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
241
241
|
}
|
|
242
242
|
}
|
|
243
243
|
|
|
244
|
+
async getAllSymbols(): Promise<Symbols> {
|
|
245
|
+
//get hash with all reserved symbol ranges
|
|
246
|
+
const rangeKey = this.mintKey(KeyType.SYMKEYS, { appId: this.appId });
|
|
247
|
+
const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
|
|
248
|
+
const rangeKeys = Object.keys(ranges).sort();
|
|
249
|
+
delete rangeKeys[':cursor'];
|
|
250
|
+
const multi = this.getMulti();
|
|
251
|
+
for (const rangeKey of rangeKeys) {
|
|
252
|
+
const symbolKey = this.mintKey(KeyType.SYMKEYS, { activityId: rangeKey, appId: this.appId });
|
|
253
|
+
multi[this.commands.hgetall](symbolKey);
|
|
254
|
+
}
|
|
255
|
+
const results = await multi.exec() as Array<[null, Symbols]> | Array<Symbols>;
|
|
256
|
+
|
|
257
|
+
const symbolSets: Symbols = {};
|
|
258
|
+
results.forEach((result: [null, Symbols] | Symbols, index: number) => {
|
|
259
|
+
if (result) {
|
|
260
|
+
let vals: Symbols;
|
|
261
|
+
if (Array.isArray(result) && result.length === 2) {
|
|
262
|
+
vals = result[1];
|
|
263
|
+
} else {
|
|
264
|
+
vals = result as Symbols;
|
|
265
|
+
}
|
|
266
|
+
for (const [key, value] of Object.entries(vals)) {
|
|
267
|
+
symbolSets[value as string] = key.startsWith(rangeKeys[index]) ? key : `${rangeKeys[index]}/${key}`;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
});
|
|
271
|
+
return symbolSets;
|
|
272
|
+
}
|
|
273
|
+
|
|
244
274
|
async getSymbols(activityId: string): Promise<Symbols> {
|
|
245
275
|
let symbols: Symbols = this.cache.getSymbols(this.appId, activityId);
|
|
246
276
|
if (symbols) {
|
|
@@ -396,7 +426,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
396
426
|
* when `originJobId` is interrupted/expired, the items in the
|
|
397
427
|
* list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
|
|
398
428
|
*/
|
|
399
|
-
async registerJobDependency(originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
|
|
429
|
+
async registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
|
|
400
430
|
const privateMulti = multi || this.getMulti();
|
|
401
431
|
const dependencyParams = {
|
|
402
432
|
appId: this.appId,
|
|
@@ -406,8 +436,8 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
406
436
|
KeyType.JOB_DEPENDENTS,
|
|
407
437
|
dependencyParams,
|
|
408
438
|
);
|
|
409
|
-
//
|
|
410
|
-
const expireTask =
|
|
439
|
+
//items listed as job dependencies have different relationships
|
|
440
|
+
const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
|
|
411
441
|
privateMulti[this.commands.rpush](depKey, expireTask);
|
|
412
442
|
if (!multi) {
|
|
413
443
|
return await privateMulti.exec();
|
|
@@ -589,6 +619,15 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
589
619
|
}
|
|
590
620
|
}
|
|
591
621
|
|
|
622
|
+
async getRaw(jobId: string): Promise<StringStringType> {
|
|
623
|
+
const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
624
|
+
const job = await this.redisClient[this.commands.hgetall](jobKey);
|
|
625
|
+
if (!job) {
|
|
626
|
+
throw new GetStateError(jobId);
|
|
627
|
+
}
|
|
628
|
+
return job;
|
|
629
|
+
}
|
|
630
|
+
|
|
592
631
|
/**
|
|
593
632
|
* collate is a generic method for incrementing a value in a hash
|
|
594
633
|
* in order to track their progress during processing.
|
|
@@ -839,6 +878,12 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
839
878
|
await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
|
|
840
879
|
}
|
|
841
880
|
|
|
881
|
+
async getDependencies(jobId: string): Promise<string[]> {
|
|
882
|
+
const depParams = { appId: this.appId, jobId };
|
|
883
|
+
const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
|
|
884
|
+
return this.redisClient[this.commands.lrange](depKey, 0, -1);
|
|
885
|
+
}
|
|
886
|
+
|
|
842
887
|
/**
|
|
843
888
|
* registers a hook activity to be awakened (uses ZSET to
|
|
844
889
|
* store the 'sleep group' and LIST to store the events
|
|
@@ -862,11 +907,15 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
862
907
|
let [pType, pKey] = this.resolveTaskKeyContext(listKey);
|
|
863
908
|
const timeEvent = await this.redisClient[this.commands.lpop](pKey);
|
|
864
909
|
if (timeEvent) {
|
|
865
|
-
//there are
|
|
866
|
-
//1) sleep (awaken), 2) expire, 3) interrupt, 4) delist
|
|
867
|
-
|
|
910
|
+
//there are task types
|
|
911
|
+
//1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
|
|
912
|
+
let [type, activityId, gId, ...jobId] = timeEvent.split('::');
|
|
868
913
|
if (type === 'delist') {
|
|
869
914
|
pType = 'delist';
|
|
915
|
+
} else if (type === 'child') {
|
|
916
|
+
pType = 'child';
|
|
917
|
+
} else if (type === 'expire-child') {
|
|
918
|
+
type = 'expire'; //use the same logic as 'expire'
|
|
870
919
|
}
|
|
871
920
|
return [listKey, jobId.join('::'), gId, activityId, pType];
|
|
872
921
|
}
|
|
@@ -884,7 +933,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
884
933
|
* generic LIST (lists typically contain target job ids)
|
|
885
934
|
* @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
|
|
886
935
|
*/
|
|
887
|
-
resolveTaskKeyContext(listKey: string): [
|
|
936
|
+
resolveTaskKeyContext(listKey: string): [WorkListTaskType, string] {
|
|
888
937
|
if (listKey.startsWith('::INTERRUPT')) {
|
|
889
938
|
return ['interrupt', listKey.split('::')[2]];
|
|
890
939
|
} else if (listKey.startsWith('::EXPIRE')) {
|
package/services/task/index.ts
CHANGED
|
@@ -117,7 +117,10 @@ class TaskService {
|
|
|
117
117
|
|
|
118
118
|
if (Array.isArray(workListTask)) {
|
|
119
119
|
const [listKey, target, gId, activityId, type] = workListTask;
|
|
120
|
-
if (type === '
|
|
120
|
+
if (type === 'child') {
|
|
121
|
+
//continue; this child is listed here for convenience, but
|
|
122
|
+
// will be expired by an origin ancestor and is listed there
|
|
123
|
+
} else if (type === 'delist') {
|
|
121
124
|
//delist the signalKey (target)
|
|
122
125
|
const key = this.store.mintKey(KeyType.SIGNALS, { appId: this.store.appId });
|
|
123
126
|
await this.store.redisClient[this.store.commands.hdel](key, target);
|
package/services/worker/index.ts
CHANGED
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { KeyType } from "../../modules/key";
|
|
2
|
+
import { formatISODate, identifyRedisType } from "../../modules/utils";
|
|
3
|
+
import { ConnectorService } from "../connector";
|
|
2
4
|
import { ILogger } from "../logger";
|
|
3
5
|
import { Router } from "../router";
|
|
4
6
|
import { StoreService } from '../store';
|
|
5
|
-
import { RedisStoreService as RedisStore } from '../store/clients/redis';
|
|
6
7
|
import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
|
|
8
|
+
import { RedisStoreService as RedisStore } from '../store/clients/redis';
|
|
7
9
|
import { StreamService } from '../stream';
|
|
8
|
-
import { RedisStreamService as RedisStream } from '../stream/clients/redis';
|
|
9
10
|
import { IORedisStreamService as IORedisStream } from '../stream/clients/ioredis';
|
|
11
|
+
import { RedisStreamService as RedisStream } from '../stream/clients/redis';
|
|
10
12
|
import { SubService } from '../sub';
|
|
11
13
|
import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
|
|
12
14
|
import { RedisSubService as RedisSub } from '../sub/clients/redis';
|
|
13
|
-
import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
|
|
14
15
|
import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
|
|
16
|
+
import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
|
|
15
17
|
import {
|
|
16
18
|
QuorumMessage,
|
|
17
19
|
QuorumProfile,
|
|
@@ -19,8 +21,6 @@ import {
|
|
|
19
21
|
import { RedisClient, RedisMulti } from "../../types/redis";
|
|
20
22
|
import { RedisClientType } from '../../types/redisclient';
|
|
21
23
|
import { StreamRole } from "../../types/stream";
|
|
22
|
-
import { identifyRedisType } from "../../modules/utils";
|
|
23
|
-
import { ConnectorService } from "../connector";
|
|
24
24
|
|
|
25
25
|
class WorkerService {
|
|
26
26
|
namespace: string;
|
|
@@ -174,6 +174,8 @@ class WorkerService {
|
|
|
174
174
|
app_id: this.appId,
|
|
175
175
|
worker_topic: this.topic,
|
|
176
176
|
stream: this.stream.mintKey(KeyType.STREAMS, params),
|
|
177
|
+
counts: this.router.counts,
|
|
178
|
+
timestamp: formatISODate(new Date()),
|
|
177
179
|
};
|
|
178
180
|
}
|
|
179
181
|
this.store.publish(
|
package/types/activity.ts
CHANGED
|
@@ -38,13 +38,18 @@ interface Measure {
|
|
|
38
38
|
|
|
39
39
|
interface TriggerActivityStats {
|
|
40
40
|
/**
|
|
41
|
-
* parent job; including this allows the parent's
|
|
41
|
+
* dependent parent job id; including this allows the parent's
|
|
42
42
|
* expiration/interruption events to cascade; set
|
|
43
43
|
* `expire` in the YAML for the dependent graph
|
|
44
44
|
* to 0 and provide the parent for dependent,
|
|
45
45
|
* cascading interruption and cleanup
|
|
46
46
|
*/
|
|
47
47
|
parent?: string;
|
|
48
|
+
/**
|
|
49
|
+
* adjacent parent job id; this is the actual adjacent
|
|
50
|
+
* parent in the graph, but it is not used for cascading expiration
|
|
51
|
+
*/
|
|
52
|
+
adjacent?: string;
|
|
48
53
|
id?: { [key: string]: unknown } | string;
|
|
49
54
|
key?: { [key: string]: unknown } | string;
|
|
50
55
|
/**
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { StringAnyType, StringStringType } from "./serializer";
|
|
2
|
+
|
|
3
|
+
export type ExportItem = [(string | null), string, any];
|
|
4
|
+
|
|
5
|
+
export interface ExportOptions {};
|
|
6
|
+
|
|
7
|
+
export type JobAction = {
|
|
8
|
+
cursor: number;
|
|
9
|
+
items: ExportItem[];
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
export interface JobActionExport {
|
|
13
|
+
hooks: {
|
|
14
|
+
[key: string]: JobAction;
|
|
15
|
+
};
|
|
16
|
+
main: JobAction;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export interface ActivityAction {
|
|
20
|
+
action: string;
|
|
21
|
+
target: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface JobTimeline {
|
|
25
|
+
activity: string; //activity name
|
|
26
|
+
dimension: string; //dimensional isolate path
|
|
27
|
+
duplex: 'entry' | 'exit'; //activity entry or exit
|
|
28
|
+
timestamp: string; //actually a number but too many digits for JS
|
|
29
|
+
actions?: ActivityAction[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface DependencyExport {
|
|
33
|
+
type: string;
|
|
34
|
+
topic: string;
|
|
35
|
+
gid: string;
|
|
36
|
+
jid: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ExportTransitions {
|
|
40
|
+
[key: string]: string[];
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export interface ExportCycles {
|
|
44
|
+
[key: string]: string[];
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export interface DurableJobExport {
|
|
48
|
+
data: StringAnyType;
|
|
49
|
+
dependencies: DependencyExport[];
|
|
50
|
+
state: StringAnyType;
|
|
51
|
+
status: string;
|
|
52
|
+
timeline: JobTimeline[];
|
|
53
|
+
transitions: ExportTransitions;
|
|
54
|
+
cycles: ExportCycles;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export interface JobExport {
|
|
58
|
+
dependencies: DependencyExport[];
|
|
59
|
+
process: StringAnyType;
|
|
60
|
+
status: string;
|
|
61
|
+
};
|
package/types/index.ts
CHANGED
|
@@ -53,7 +53,19 @@ export {
|
|
|
53
53
|
WorkflowSearchOptions,
|
|
54
54
|
WorkflowDataType,
|
|
55
55
|
WorkflowOptions,
|
|
56
|
-
} from './durable'
|
|
56
|
+
} from './durable';
|
|
57
|
+
export {
|
|
58
|
+
ActivityAction,
|
|
59
|
+
DependencyExport,
|
|
60
|
+
DurableJobExport,
|
|
61
|
+
ExportCycles,
|
|
62
|
+
ExportItem,
|
|
63
|
+
ExportOptions,
|
|
64
|
+
ExportTransitions,
|
|
65
|
+
JobAction,
|
|
66
|
+
JobExport,
|
|
67
|
+
JobActionExport,
|
|
68
|
+
JobTimeline } from './exporter';
|
|
57
69
|
export {
|
|
58
70
|
HookCondition,
|
|
59
71
|
HookConditions,
|
package/types/quorum.ts
CHANGED
package/types/task.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export type WorkListTaskType = 'sleep' | 'expire' | 'interrupt' | 'delist';
|
|
1
|
+
export type WorkListTaskType = 'sleep' | 'expire' | 'expire-child' | 'interrupt' | 'delist' | 'child';
|