@hotmeshio/hotmesh 0.0.43 → 0.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/package.json +1 -1
- package/build/services/activities/trigger.js +7 -1
- package/build/services/durable/exporter.d.ts +105 -0
- package/build/services/durable/exporter.js +374 -0
- package/build/services/durable/factory.js +6 -63
- package/build/services/durable/handle.d.ts +4 -0
- package/build/services/durable/handle.js +5 -0
- package/build/services/durable/workflow.js +24 -21
- package/build/services/engine/index.d.ts +6 -1
- package/build/services/engine/index.js +9 -2
- package/build/services/exporter/index.d.ts +46 -0
- package/build/services/exporter/index.js +126 -0
- package/build/services/hotmesh/index.d.ts +4 -1
- package/build/services/hotmesh/index.js +6 -0
- package/build/services/quorum/index.js +2 -1
- package/build/services/router/index.d.ts +3 -0
- package/build/services/router/index.js +3 -0
- package/build/services/store/index.d.ts +5 -2
- package/build/services/store/index.js +54 -6
- package/build/services/task/index.js +5 -1
- package/build/services/worker/index.js +5 -4
- package/build/types/activity.d.ts +6 -1
- package/build/types/exporter.d.ts +51 -0
- package/build/types/exporter.js +8 -0
- package/build/types/index.d.ts +1 -0
- package/build/types/quorum.d.ts +1 -0
- package/build/types/task.d.ts +1 -1
- package/package.json +1 -1
- package/services/activities/trigger.ts +14 -0
- package/services/durable/exporter.ts +408 -0
- package/services/durable/factory.ts +6 -63
- package/services/durable/handle.ts +12 -0
- package/services/durable/workflow.ts +24 -22
- package/services/engine/index.ts +20 -5
- package/services/exporter/index.ts +147 -0
- package/services/hotmesh/index.ts +8 -1
- package/services/quorum/index.ts +2 -1
- package/services/router/index.ts +3 -0
- package/services/store/index.ts +56 -7
- package/services/task/index.ts +4 -1
- package/services/worker/index.ts +6 -5
- package/types/activity.ts +6 -1
- package/types/exporter.ts +61 -0
- package/types/index.ts +13 -1
- package/types/quorum.ts +1 -0
- package/types/task.ts +1 -1
|
@@ -37,7 +37,7 @@ class WorkflowService {
|
|
|
37
37
|
const entityOrEmptyString = options.entity ?? '';
|
|
38
38
|
//If the workflowId is not provided, it is generated from the entity and the workflow name
|
|
39
39
|
const childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
|
|
40
|
-
const parentWorkflowId =
|
|
40
|
+
const parentWorkflowId = workflowId;
|
|
41
41
|
const client = new client_1.ClientService({
|
|
42
42
|
connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
|
|
43
43
|
});
|
|
@@ -75,32 +75,35 @@ class WorkflowService {
|
|
|
75
75
|
const workflowSpan = store.get('workflowSpan');
|
|
76
76
|
const COUNTER = store.get('counter');
|
|
77
77
|
const execIndex = COUNTER.counter = COUNTER.counter + 1;
|
|
78
|
+
const sessionId = `-start${workflowDimension}-${execIndex}-`;
|
|
78
79
|
//NOTE: this is the hash prefix; necessary for the search index to locate the entity
|
|
79
80
|
const entityOrEmptyString = options.entity ?? '';
|
|
80
81
|
//If the workflowId is not provided, it is generated from the entity and the workflow name
|
|
81
|
-
const
|
|
82
|
-
const parentWorkflowId = `${workflowId}-f`;
|
|
82
|
+
const parentWorkflowId = workflowId;
|
|
83
83
|
const workflowTopic = `${options.entity ?? options.taskQueue}-${options.entity ?? options.workflowName}`;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
|
|
85
|
+
const keyParams = { appId: hotMeshClient.appId, jobId: workflowId };
|
|
86
|
+
const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
|
|
87
|
+
let childJobId = await hotMeshClient.engine.store.exec('HGET', workflowGuid, sessionId);
|
|
88
|
+
if (childJobId) {
|
|
88
89
|
return childJobId;
|
|
89
90
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
|
|
93
|
-
});
|
|
94
|
-
await client.workflow.start({
|
|
95
|
-
...options,
|
|
96
|
-
namespace,
|
|
97
|
-
workflowId: childJobId,
|
|
98
|
-
parentWorkflowId,
|
|
99
|
-
workflowTrace,
|
|
100
|
-
workflowSpan,
|
|
101
|
-
});
|
|
102
|
-
return childJobId;
|
|
91
|
+
else {
|
|
92
|
+
childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
|
|
103
93
|
}
|
|
94
|
+
const client = new client_1.ClientService({
|
|
95
|
+
connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
|
|
96
|
+
});
|
|
97
|
+
await client.workflow.start({
|
|
98
|
+
...options,
|
|
99
|
+
namespace,
|
|
100
|
+
workflowId: childJobId,
|
|
101
|
+
parentWorkflowId,
|
|
102
|
+
workflowTrace,
|
|
103
|
+
workflowSpan,
|
|
104
|
+
});
|
|
105
|
+
await hotMeshClient.engine.store.exec('HSET', workflowGuid, sessionId, childJobId);
|
|
106
|
+
return childJobId;
|
|
104
107
|
}
|
|
105
108
|
/**
|
|
106
109
|
* Wraps activities in a proxy that will durably run them
|
|
@@ -439,7 +442,7 @@ class WorkflowService {
|
|
|
439
442
|
arguments: Array.from(arguments),
|
|
440
443
|
//when the origin job is removed
|
|
441
444
|
originJobId: originJobId ?? workflowId,
|
|
442
|
-
parentWorkflowId:
|
|
445
|
+
parentWorkflowId: workflowId,
|
|
443
446
|
workflowId: activityJobId,
|
|
444
447
|
workflowTopic: activityTopic,
|
|
445
448
|
activityName,
|
|
@@ -5,6 +5,7 @@ import { Interrupt } from '../activities/interrupt';
|
|
|
5
5
|
import { Signal } from '../activities/signal';
|
|
6
6
|
import { Worker } from '../activities/worker';
|
|
7
7
|
import { Trigger } from '../activities/trigger';
|
|
8
|
+
import { ExporterService } from '../exporter';
|
|
8
9
|
import { ILogger } from '../logger';
|
|
9
10
|
import { Router } from '../router';
|
|
10
11
|
import { StoreService } from '../store';
|
|
@@ -18,15 +19,17 @@ import { JobState, JobData, JobMetadata, JobOutput, JobStatus, JobInterruptOptio
|
|
|
18
19
|
import { HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshSettings } from '../../types/hotmesh';
|
|
19
20
|
import { JobMessageCallback } from '../../types/quorum';
|
|
20
21
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
21
|
-
import { StringAnyType } from '../../types/serializer';
|
|
22
|
+
import { StringAnyType, StringStringType } from '../../types/serializer';
|
|
22
23
|
import { GetStatsOptions, IdsResponse, JobStatsInput, StatsResponse } from '../../types/stats';
|
|
23
24
|
import { StreamCode, StreamData, StreamDataResponse, StreamError, StreamStatus } from '../../types/stream';
|
|
24
25
|
import { WorkListTaskType } from '../../types/task';
|
|
26
|
+
import { JobExport } from '../../types/exporter';
|
|
25
27
|
declare class EngineService {
|
|
26
28
|
namespace: string;
|
|
27
29
|
apps: HotMeshApps | null;
|
|
28
30
|
appId: string;
|
|
29
31
|
guid: string;
|
|
32
|
+
exporter: ExporterService | null;
|
|
30
33
|
router: Router | null;
|
|
31
34
|
store: StoreService<RedisClient, RedisMulti> | null;
|
|
32
35
|
stream: StreamService<RedisClient, RedisMulti> | null;
|
|
@@ -88,6 +91,8 @@ declare class EngineService {
|
|
|
88
91
|
* it will be expired immediately.
|
|
89
92
|
*/
|
|
90
93
|
resolveExpires(context: JobState, options: JobCompletionOptions): number;
|
|
94
|
+
export(jobId: string): Promise<JobExport>;
|
|
95
|
+
getRaw(jobId: string): Promise<StringStringType>;
|
|
91
96
|
getStatus(jobId: string): Promise<JobStatus>;
|
|
92
97
|
getState(topic: string, jobId: string): Promise<JobOutput>;
|
|
93
98
|
getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
|
|
@@ -9,6 +9,7 @@ const enums_1 = require("../../modules/enums");
|
|
|
9
9
|
const utils_1 = require("../../modules/utils");
|
|
10
10
|
const activities_1 = __importDefault(require("../activities"));
|
|
11
11
|
const compiler_1 = require("../compiler");
|
|
12
|
+
const exporter_1 = require("../exporter");
|
|
12
13
|
const reporter_1 = require("../reporter");
|
|
13
14
|
const router_1 = require("../router");
|
|
14
15
|
const serializer_1 = require("../serializer");
|
|
@@ -41,8 +42,8 @@ class EngineService {
|
|
|
41
42
|
await instance.initStreamChannel(config.engine.stream);
|
|
42
43
|
instance.router = instance.initRouter(config);
|
|
43
44
|
instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
|
|
44
|
-
//the task service is used by the engine to process `webhooks` and `timehooks`
|
|
45
45
|
instance.taskService = new task_1.TaskService(instance.store, logger);
|
|
46
|
+
instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
|
|
46
47
|
return instance;
|
|
47
48
|
}
|
|
48
49
|
}
|
|
@@ -526,9 +527,15 @@ class EngineService {
|
|
|
526
527
|
return options.expire ?? context.metadata.expire ?? enums_1.HMSH_EXPIRE_JOB_SECONDS;
|
|
527
528
|
}
|
|
528
529
|
// ****** GET JOB STATE/COLLATION STATUS BY ID *********
|
|
530
|
+
async export(jobId) {
|
|
531
|
+
return await this.exporter.export(jobId);
|
|
532
|
+
}
|
|
533
|
+
async getRaw(jobId) {
|
|
534
|
+
return await this.store.getRaw(jobId);
|
|
535
|
+
}
|
|
529
536
|
async getStatus(jobId) {
|
|
530
537
|
const { id: appId } = await this.getVID();
|
|
531
|
-
return this.store.getStatus(jobId, appId);
|
|
538
|
+
return await this.store.getStatus(jobId, appId);
|
|
532
539
|
}
|
|
533
540
|
//todo: add 'options' parameter;
|
|
534
541
|
// (e.g, if {dimensions:true}, use hscan to deliver
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { ILogger } from '../logger';
|
|
2
|
+
import { StoreService } from '../store';
|
|
3
|
+
import { StringStringType, Symbols } from "../../types/serializer";
|
|
4
|
+
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
5
|
+
import { DependencyExport, ExportOptions, JobActionExport, JobExport } from '../../types/exporter';
|
|
6
|
+
import { SerializerService } from '../serializer';
|
|
7
|
+
/**
|
|
8
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
9
|
+
* Expands process data and includes dependency list
|
|
10
|
+
*/
|
|
11
|
+
declare class ExporterService {
|
|
12
|
+
appId: string;
|
|
13
|
+
logger: ILogger;
|
|
14
|
+
serializer: SerializerService;
|
|
15
|
+
store: StoreService<RedisClient, RedisMulti>;
|
|
16
|
+
symbols: Promise<Symbols> | Symbols;
|
|
17
|
+
constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
|
|
18
|
+
/**
|
|
19
|
+
* Convert the job hash and dependency list into a JobExport object.
|
|
20
|
+
* This object contains various facets that describe the interaction
|
|
21
|
+
* in terms relevant to narrative storytelling.
|
|
22
|
+
*/
|
|
23
|
+
export(jobId: string, options?: ExportOptions): Promise<JobExport>;
|
|
24
|
+
/**
|
|
25
|
+
* Inflates the key from Redis, 3-character symbol
|
|
26
|
+
* into a human-readable JSON path, reflecting the
|
|
27
|
+
* tree-like structure of the unidimensional Hash
|
|
28
|
+
*/
|
|
29
|
+
inflateKey(key: string): string;
|
|
30
|
+
/**
|
|
31
|
+
* Inflates the job data from Redis into a JobExport object
|
|
32
|
+
* @param jobHash - the job data from Redis
|
|
33
|
+
* @param dependencyList - the list of dependencies for the job
|
|
34
|
+
* @returns - the inflated job data
|
|
35
|
+
*/
|
|
36
|
+
inflate(jobHash: StringStringType, dependencyList: string[]): JobExport;
|
|
37
|
+
/**
|
|
38
|
+
* Inflates the dependency data from Redis into a JobExport object by
|
|
39
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
40
|
+
* into a story
|
|
41
|
+
* @param data - the dependency data from Redis
|
|
42
|
+
* @returns - the organized dependency data
|
|
43
|
+
*/
|
|
44
|
+
inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
|
|
45
|
+
}
|
|
46
|
+
export { ExporterService };
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ExporterService = void 0;
|
|
4
|
+
const serializer_1 = require("../serializer");
|
|
5
|
+
const utils_1 = require("../../modules/utils");
|
|
6
|
+
/**
|
|
7
|
+
* Downloads job data from Redis (hscan, hmget, hgetall)
|
|
8
|
+
* Expands process data and includes dependency list
|
|
9
|
+
*/
|
|
10
|
+
class ExporterService {
|
|
11
|
+
constructor(appId, store, logger) {
|
|
12
|
+
this.appId = appId;
|
|
13
|
+
this.logger = logger;
|
|
14
|
+
this.store = store;
|
|
15
|
+
this.serializer = new serializer_1.SerializerService();
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Convert the job hash and dependency list into a JobExport object.
|
|
19
|
+
* This object contains various facets that describe the interaction
|
|
20
|
+
* in terms relevant to narrative storytelling.
|
|
21
|
+
*/
|
|
22
|
+
async export(jobId, options = {}) {
|
|
23
|
+
if (!this.symbols) {
|
|
24
|
+
this.symbols = this.store.getAllSymbols();
|
|
25
|
+
this.symbols = await this.symbols;
|
|
26
|
+
}
|
|
27
|
+
const depData = await this.store.getDependencies(jobId);
|
|
28
|
+
const jobData = await this.store.getRaw(jobId);
|
|
29
|
+
const jobExport = this.inflate(jobData, depData);
|
|
30
|
+
return jobExport;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Inflates the key from Redis, 3-character symbol
|
|
34
|
+
* into a human-readable JSON path, reflecting the
|
|
35
|
+
* tree-like structure of the unidimensional Hash
|
|
36
|
+
*/
|
|
37
|
+
inflateKey(key) {
|
|
38
|
+
return (key in this.symbols) ? this.symbols[key] : key;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Inflates the job data from Redis into a JobExport object
|
|
42
|
+
* @param jobHash - the job data from Redis
|
|
43
|
+
* @param dependencyList - the list of dependencies for the job
|
|
44
|
+
* @returns - the inflated job data
|
|
45
|
+
*/
|
|
46
|
+
inflate(jobHash, dependencyList) {
|
|
47
|
+
//the list of actions taken in the workflow and hook functions
|
|
48
|
+
const actions = {
|
|
49
|
+
hooks: {},
|
|
50
|
+
main: {
|
|
51
|
+
cursor: -1,
|
|
52
|
+
items: []
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
const process = {};
|
|
56
|
+
const dependencies = this.inflateDependencyData(dependencyList, actions);
|
|
57
|
+
const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
|
|
58
|
+
Object.entries(jobHash).forEach(([key, value]) => {
|
|
59
|
+
const match = key.match(regex);
|
|
60
|
+
if (match) {
|
|
61
|
+
//activity process state
|
|
62
|
+
const [_, letters, numbers] = match;
|
|
63
|
+
const path = this.inflateKey(letters);
|
|
64
|
+
const dimensions = `${numbers.replace(/,/g, '/')}`;
|
|
65
|
+
const resolved = this.serializer.fromString(value);
|
|
66
|
+
process[`${dimensions}/${path}`] = resolved;
|
|
67
|
+
}
|
|
68
|
+
else if (key.length === 3) {
|
|
69
|
+
//job state
|
|
70
|
+
process[this.inflateKey(key)] = this.serializer.fromString(value);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
return {
|
|
74
|
+
dependencies,
|
|
75
|
+
process: (0, utils_1.restoreHierarchy)(process),
|
|
76
|
+
status: jobHash[':'],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Inflates the dependency data from Redis into a JobExport object by
|
|
81
|
+
* organizing the dimensional isolate in sch a way asto interleave
|
|
82
|
+
* into a story
|
|
83
|
+
* @param data - the dependency data from Redis
|
|
84
|
+
* @returns - the organized dependency data
|
|
85
|
+
*/
|
|
86
|
+
inflateDependencyData(data, actions) {
|
|
87
|
+
const hookReg = /([0-9,]+)-(\d+)$/;
|
|
88
|
+
const flowReg = /-(\d+)$/;
|
|
89
|
+
return data.map((dependency, index) => {
|
|
90
|
+
const [action, topic, gid, ...jid] = dependency.split('::');
|
|
91
|
+
const jobId = jid.join('::');
|
|
92
|
+
const match = jobId.match(hookReg);
|
|
93
|
+
let prefix;
|
|
94
|
+
let type;
|
|
95
|
+
let dimensionKey = '';
|
|
96
|
+
if (match) {
|
|
97
|
+
//hook-originating dependency
|
|
98
|
+
const [_, dimension, counter] = match;
|
|
99
|
+
dimensionKey = dimension.split(',').join('/');
|
|
100
|
+
prefix = `${dimensionKey}[${counter}]`;
|
|
101
|
+
type = 'hook';
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
const match = jobId.match(flowReg);
|
|
105
|
+
if (match) {
|
|
106
|
+
//main workflow-originating dependency
|
|
107
|
+
const [_, counter] = match;
|
|
108
|
+
prefix = `[${counter}]`;
|
|
109
|
+
type = 'flow';
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
//'other' types like signal cleanup
|
|
113
|
+
prefix = '/';
|
|
114
|
+
type = 'other';
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
type: action,
|
|
119
|
+
topic,
|
|
120
|
+
gid,
|
|
121
|
+
jid: jobId,
|
|
122
|
+
};
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
exports.ExporterService = ExporterService;
|
|
@@ -7,7 +7,8 @@ import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
|
|
|
7
7
|
import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
|
|
8
8
|
import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
|
|
9
9
|
import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
|
|
10
|
-
import { StringAnyType } from '../../types/serializer';
|
|
10
|
+
import { StringAnyType, StringStringType } from '../../types/serializer';
|
|
11
|
+
import { JobExport } from '../../types/exporter';
|
|
11
12
|
declare class HotMeshService {
|
|
12
13
|
namespace: string;
|
|
13
14
|
appId: string;
|
|
@@ -35,6 +36,8 @@ declare class HotMeshService {
|
|
|
35
36
|
plan(path: string): Promise<HotMeshManifest>;
|
|
36
37
|
deploy(pathOrYAML: string): Promise<HotMeshManifest>;
|
|
37
38
|
activate(version: string, delay?: number): Promise<boolean>;
|
|
39
|
+
export(jobId: string): Promise<JobExport>;
|
|
40
|
+
getRaw(jobId: string): Promise<StringStringType>;
|
|
38
41
|
getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
|
|
39
42
|
getStatus(jobId: string): Promise<JobStatus>;
|
|
40
43
|
getState(topic: string, jobId: string): Promise<JobOutput>;
|
|
@@ -104,6 +104,12 @@ class HotMeshService {
|
|
|
104
104
|
return await this.quorum?.activate(version, delay);
|
|
105
105
|
}
|
|
106
106
|
// ************* REPORTER METHODS *************
|
|
107
|
+
async export(jobId) {
|
|
108
|
+
return await this.engine?.export(jobId);
|
|
109
|
+
}
|
|
110
|
+
async getRaw(jobId) {
|
|
111
|
+
return await this.engine?.getRaw(jobId);
|
|
112
|
+
}
|
|
107
113
|
async getStats(topic, query) {
|
|
108
114
|
return await this.engine?.getStats(topic, query);
|
|
109
115
|
}
|
|
@@ -103,7 +103,8 @@ class QuorumService {
|
|
|
103
103
|
engine_id: this.guid,
|
|
104
104
|
namespace: this.namespace,
|
|
105
105
|
app_id: this.appId,
|
|
106
|
-
stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId })
|
|
106
|
+
stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId }),
|
|
107
|
+
counts: this.engine.router.counts,
|
|
107
108
|
};
|
|
108
109
|
}
|
|
109
110
|
this.store.publish(hotmesh_1.KeyType.QUORUM, {
|
|
@@ -17,6 +17,9 @@ declare class Router {
|
|
|
17
17
|
logger: ILogger;
|
|
18
18
|
throttle: number;
|
|
19
19
|
errorCount: number;
|
|
20
|
+
counts: {
|
|
21
|
+
[key: string]: number;
|
|
22
|
+
};
|
|
20
23
|
currentTimerId: NodeJS.Timeout | null;
|
|
21
24
|
shouldConsume: boolean;
|
|
22
25
|
constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
|
|
@@ -10,6 +10,7 @@ class Router {
|
|
|
10
10
|
constructor(config, stream, store, logger) {
|
|
11
11
|
this.throttle = 0;
|
|
12
12
|
this.errorCount = 0;
|
|
13
|
+
this.counts = {};
|
|
13
14
|
this.currentTimerId = null;
|
|
14
15
|
this.appId = config.appId;
|
|
15
16
|
this.guid = config.guid;
|
|
@@ -146,6 +147,8 @@ class Router {
|
|
|
146
147
|
else {
|
|
147
148
|
output.metadata.guid = (0, utils_1.guid)();
|
|
148
149
|
}
|
|
150
|
+
const code = output.code || 200;
|
|
151
|
+
this.counts[code] = (this.counts[code] || 0) + 1;
|
|
149
152
|
output.type = stream_1.StreamDataType.RESPONSE;
|
|
150
153
|
return await this.publishMessage(null, output);
|
|
151
154
|
}
|
|
@@ -51,6 +51,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
51
51
|
getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
|
|
52
52
|
setSettings(manifest: HotMeshSettings): Promise<any>;
|
|
53
53
|
reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;
|
|
54
|
+
getAllSymbols(): Promise<Symbols>;
|
|
54
55
|
getSymbols(activityId: string): Promise<Symbols>;
|
|
55
56
|
addSymbols(activityId: string, symbols: Symbols): Promise<boolean>;
|
|
56
57
|
seedSymbols(target: string, type: 'JOB' | 'ACTIVITY', startIndex: number): StringStringType;
|
|
@@ -68,7 +69,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
68
69
|
* when `originJobId` is interrupted/expired, the items in the
|
|
69
70
|
* list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
|
|
70
71
|
*/
|
|
71
|
-
registerJobDependency(originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
|
|
72
|
+
registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
|
|
72
73
|
/**
|
|
73
74
|
* Ensures a `hook signal` is delisted when its parent activity/job
|
|
74
75
|
* is interrupted/expired.
|
|
@@ -87,6 +88,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
87
88
|
*/
|
|
88
89
|
getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
|
|
89
90
|
getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
|
|
91
|
+
getRaw(jobId: string): Promise<StringStringType>;
|
|
90
92
|
/**
|
|
91
93
|
* collate is a generic method for incrementing a value in a hash
|
|
92
94
|
* in order to track their progress during processing.
|
|
@@ -124,6 +126,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
124
126
|
* is a standard `expire` or an `interrupt`
|
|
125
127
|
*/
|
|
126
128
|
registerDependenciesForCleanup(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void>;
|
|
129
|
+
getDependencies(jobId: string): Promise<string[]>;
|
|
127
130
|
/**
|
|
128
131
|
* registers a hook activity to be awakened (uses ZSET to
|
|
129
132
|
* store the 'sleep group' and LIST to store the events
|
|
@@ -140,7 +143,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
140
143
|
* generic LIST (lists typically contain target job ids)
|
|
141
144
|
* @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
|
|
142
145
|
*/
|
|
143
|
-
resolveTaskKeyContext(listKey: string): [
|
|
146
|
+
resolveTaskKeyContext(listKey: string): [WorkListTaskType, string];
|
|
144
147
|
/**
|
|
145
148
|
* Interrupts a job and sets sets a job error (410), if 'throw'!=false.
|
|
146
149
|
* This method is called by the engine and not by an activity and is
|
|
@@ -164,6 +164,35 @@ class StoreService {
|
|
|
164
164
|
return [actualLowerLimit, upperLimit, symbols];
|
|
165
165
|
}
|
|
166
166
|
}
|
|
167
|
+
async getAllSymbols() {
|
|
168
|
+
//get hash with all reserved symbol ranges
|
|
169
|
+
const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
|
|
170
|
+
const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
|
|
171
|
+
const rangeKeys = Object.keys(ranges).sort();
|
|
172
|
+
delete rangeKeys[':cursor'];
|
|
173
|
+
const multi = this.getMulti();
|
|
174
|
+
for (const rangeKey of rangeKeys) {
|
|
175
|
+
const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, { activityId: rangeKey, appId: this.appId });
|
|
176
|
+
multi[this.commands.hgetall](symbolKey);
|
|
177
|
+
}
|
|
178
|
+
const results = await multi.exec();
|
|
179
|
+
const symbolSets = {};
|
|
180
|
+
results.forEach((result, index) => {
|
|
181
|
+
if (result) {
|
|
182
|
+
let vals;
|
|
183
|
+
if (Array.isArray(result) && result.length === 2) {
|
|
184
|
+
vals = result[1];
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
vals = result;
|
|
188
|
+
}
|
|
189
|
+
for (const [key, value] of Object.entries(vals)) {
|
|
190
|
+
symbolSets[value] = key.startsWith(rangeKeys[index]) ? key : `${rangeKeys[index]}/${key}`;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
return symbolSets;
|
|
195
|
+
}
|
|
167
196
|
async getSymbols(activityId) {
|
|
168
197
|
let symbols = this.cache.getSymbols(this.appId, activityId);
|
|
169
198
|
if (symbols) {
|
|
@@ -314,15 +343,15 @@ class StoreService {
|
|
|
314
343
|
* when `originJobId` is interrupted/expired, the items in the
|
|
315
344
|
* list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
|
|
316
345
|
*/
|
|
317
|
-
async registerJobDependency(originJobId, topic, jobId, gId, multi) {
|
|
346
|
+
async registerJobDependency(depType, originJobId, topic, jobId, gId, multi) {
|
|
318
347
|
const privateMulti = multi || this.getMulti();
|
|
319
348
|
const dependencyParams = {
|
|
320
349
|
appId: this.appId,
|
|
321
350
|
jobId: originJobId,
|
|
322
351
|
};
|
|
323
352
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
|
|
324
|
-
//
|
|
325
|
-
const expireTask =
|
|
353
|
+
//items listed as job dependencies have different relationships
|
|
354
|
+
const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
|
|
326
355
|
privateMulti[this.commands.rpush](depKey, expireTask);
|
|
327
356
|
if (!multi) {
|
|
328
357
|
return await privateMulti.exec();
|
|
@@ -487,6 +516,14 @@ class StoreService {
|
|
|
487
516
|
throw new errors_1.GetStateError(jobId);
|
|
488
517
|
}
|
|
489
518
|
}
|
|
519
|
+
async getRaw(jobId) {
|
|
520
|
+
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
|
|
521
|
+
const job = await this.redisClient[this.commands.hgetall](jobKey);
|
|
522
|
+
if (!job) {
|
|
523
|
+
throw new errors_1.GetStateError(jobId);
|
|
524
|
+
}
|
|
525
|
+
return job;
|
|
526
|
+
}
|
|
490
527
|
/**
|
|
491
528
|
* collate is a generic method for incrementing a value in a hash
|
|
492
529
|
* in order to track their progress during processing.
|
|
@@ -720,6 +757,11 @@ class StoreService {
|
|
|
720
757
|
const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
|
|
721
758
|
await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
|
|
722
759
|
}
|
|
760
|
+
async getDependencies(jobId) {
|
|
761
|
+
const depParams = { appId: this.appId, jobId };
|
|
762
|
+
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
|
|
763
|
+
return this.redisClient[this.commands.lrange](depKey, 0, -1);
|
|
764
|
+
}
|
|
723
765
|
/**
|
|
724
766
|
* registers a hook activity to be awakened (uses ZSET to
|
|
725
767
|
* store the 'sleep group' and LIST to store the events
|
|
@@ -742,12 +784,18 @@ class StoreService {
|
|
|
742
784
|
let [pType, pKey] = this.resolveTaskKeyContext(listKey);
|
|
743
785
|
const timeEvent = await this.redisClient[this.commands.lpop](pKey);
|
|
744
786
|
if (timeEvent) {
|
|
745
|
-
//there are
|
|
746
|
-
//1) sleep (awaken), 2) expire, 3) interrupt, 4) delist
|
|
747
|
-
|
|
787
|
+
//there are task types
|
|
788
|
+
//1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
|
|
789
|
+
let [type, activityId, gId, ...jobId] = timeEvent.split('::');
|
|
748
790
|
if (type === 'delist') {
|
|
749
791
|
pType = 'delist';
|
|
750
792
|
}
|
|
793
|
+
else if (type === 'child') {
|
|
794
|
+
pType = 'child';
|
|
795
|
+
}
|
|
796
|
+
else if (type === 'expire-child') {
|
|
797
|
+
type = 'expire'; //use the same logic as 'expire'
|
|
798
|
+
}
|
|
751
799
|
return [listKey, jobId.join('::'), gId, activityId, pType];
|
|
752
800
|
}
|
|
753
801
|
await this.redisClient[this.commands.zrem](zsetKey, listKey);
|
|
@@ -75,7 +75,11 @@ class TaskService {
|
|
|
75
75
|
const workListTask = await this.store.getNextTask(listKey);
|
|
76
76
|
if (Array.isArray(workListTask)) {
|
|
77
77
|
const [listKey, target, gId, activityId, type] = workListTask;
|
|
78
|
-
if (type === '
|
|
78
|
+
if (type === 'child') {
|
|
79
|
+
//continue; this child is listed here for convenience, but
|
|
80
|
+
// will be expired by an origin ancestor and is listed there
|
|
81
|
+
}
|
|
82
|
+
else if (type === 'delist') {
|
|
79
83
|
//delist the signalKey (target)
|
|
80
84
|
const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, { appId: this.store.appId });
|
|
81
85
|
await this.store.redisClient[this.store.commands.hdel](key, target);
|
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.WorkerService = void 0;
|
|
4
4
|
const key_1 = require("../../modules/key");
|
|
5
|
+
const utils_1 = require("../../modules/utils");
|
|
6
|
+
const connector_1 = require("../connector");
|
|
5
7
|
const router_1 = require("../router");
|
|
6
|
-
const redis_1 = require("../store/clients/redis");
|
|
7
8
|
const ioredis_1 = require("../store/clients/ioredis");
|
|
8
|
-
const
|
|
9
|
+
const redis_1 = require("../store/clients/redis");
|
|
9
10
|
const ioredis_2 = require("../stream/clients/ioredis");
|
|
11
|
+
const redis_2 = require("../stream/clients/redis");
|
|
10
12
|
const ioredis_3 = require("../sub/clients/ioredis");
|
|
11
13
|
const redis_3 = require("../sub/clients/redis");
|
|
12
14
|
const stream_1 = require("../../types/stream");
|
|
13
|
-
const utils_1 = require("../../modules/utils");
|
|
14
|
-
const connector_1 = require("../connector");
|
|
15
15
|
class WorkerService {
|
|
16
16
|
constructor() {
|
|
17
17
|
this.reporting = false;
|
|
@@ -114,6 +114,7 @@ class WorkerService {
|
|
|
114
114
|
app_id: this.appId,
|
|
115
115
|
worker_topic: this.topic,
|
|
116
116
|
stream: this.stream.mintKey(key_1.KeyType.STREAMS, params),
|
|
117
|
+
counts: this.router.counts,
|
|
117
118
|
};
|
|
118
119
|
}
|
|
119
120
|
this.store.publish(key_1.KeyType.QUORUM, {
|
|
@@ -33,13 +33,18 @@ interface Measure {
|
|
|
33
33
|
}
|
|
34
34
|
interface TriggerActivityStats {
|
|
35
35
|
/**
|
|
36
|
-
* parent job; including this allows the parent's
|
|
36
|
+
* dependent parent job id; including this allows the parent's
|
|
37
37
|
* expiration/interruption events to cascade; set
|
|
38
38
|
* `expire` in the YAML for the dependent graph
|
|
39
39
|
* to 0 and provide the parent for dependent,
|
|
40
40
|
* cascading interruption and cleanup
|
|
41
41
|
*/
|
|
42
42
|
parent?: string;
|
|
43
|
+
/**
|
|
44
|
+
* adjacent parent job id; this is the actual adjacent
|
|
45
|
+
* parent in the graph, but it is not used for cascading expiration
|
|
46
|
+
*/
|
|
47
|
+
adjacent?: string;
|
|
43
48
|
id?: {
|
|
44
49
|
[key: string]: unknown;
|
|
45
50
|
} | string;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { StringAnyType } from "./serializer";
|
|
2
|
+
export type ExportItem = [(string | null), string, any];
|
|
3
|
+
export interface ExportOptions {
|
|
4
|
+
}
|
|
5
|
+
export type JobAction = {
|
|
6
|
+
cursor: number;
|
|
7
|
+
items: ExportItem[];
|
|
8
|
+
};
|
|
9
|
+
export interface JobActionExport {
|
|
10
|
+
hooks: {
|
|
11
|
+
[key: string]: JobAction;
|
|
12
|
+
};
|
|
13
|
+
main: JobAction;
|
|
14
|
+
}
|
|
15
|
+
export interface ActivityAction {
|
|
16
|
+
action: string;
|
|
17
|
+
target: string;
|
|
18
|
+
}
|
|
19
|
+
export interface JobTimeline {
|
|
20
|
+
activity: string;
|
|
21
|
+
dimension: string;
|
|
22
|
+
duplex: 'entry' | 'exit';
|
|
23
|
+
timestamp: string;
|
|
24
|
+
actions?: ActivityAction[];
|
|
25
|
+
}
|
|
26
|
+
export interface DependencyExport {
|
|
27
|
+
type: string;
|
|
28
|
+
topic: string;
|
|
29
|
+
gid: string;
|
|
30
|
+
jid: string;
|
|
31
|
+
}
|
|
32
|
+
export interface ExportTransitions {
|
|
33
|
+
[key: string]: string[];
|
|
34
|
+
}
|
|
35
|
+
export interface ExportCycles {
|
|
36
|
+
[key: string]: string[];
|
|
37
|
+
}
|
|
38
|
+
export interface DurableJobExport {
|
|
39
|
+
data: StringAnyType;
|
|
40
|
+
dependencies: DependencyExport[];
|
|
41
|
+
state: StringAnyType;
|
|
42
|
+
status: string;
|
|
43
|
+
timeline: JobTimeline[];
|
|
44
|
+
transitions: ExportTransitions;
|
|
45
|
+
cycles: ExportCycles;
|
|
46
|
+
}
|
|
47
|
+
export interface JobExport {
|
|
48
|
+
dependencies: DependencyExport[];
|
|
49
|
+
process: StringAnyType;
|
|
50
|
+
status: string;
|
|
51
|
+
}
|