@hotmeshio/hotmesh 0.1.15 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +623 -209
- package/build/index.d.ts +14 -3
- package/build/index.js +17 -4
- package/build/modules/enums.d.ts +12 -12
- package/build/modules/enums.js +15 -25
- package/build/modules/errors.d.ts +16 -16
- package/build/modules/errors.js +28 -28
- package/build/modules/key.d.ts +0 -37
- package/build/modules/key.js +4 -45
- package/build/modules/utils.d.ts +7 -15
- package/build/modules/utils.js +21 -44
- package/build/package.json +18 -15
- package/build/services/activities/activity.d.ts +0 -31
- package/build/services/activities/activity.js +1 -50
- package/build/services/activities/await.js +0 -4
- package/build/services/activities/cycle.d.ts +0 -7
- package/build/services/activities/cycle.js +1 -16
- package/build/services/activities/hook.d.ts +0 -6
- package/build/services/activities/hook.js +2 -12
- package/build/services/activities/interrupt.js +0 -8
- package/build/services/activities/signal.d.ts +0 -6
- package/build/services/activities/signal.js +0 -15
- package/build/services/activities/trigger.d.ts +4 -5
- package/build/services/activities/trigger.js +22 -16
- package/build/services/activities/worker.js +0 -4
- package/build/services/collator/index.d.ts +0 -70
- package/build/services/collator/index.js +1 -91
- package/build/services/compiler/deployer.js +6 -38
- package/build/services/compiler/index.d.ts +0 -15
- package/build/services/compiler/index.js +0 -20
- package/build/services/compiler/validator.d.ts +0 -3
- package/build/services/compiler/validator.js +0 -25
- package/build/services/connector/clients/ioredis.js +0 -2
- package/build/services/connector/clients/redis.js +0 -2
- package/build/services/connector/index.js +0 -2
- package/build/services/engine/index.d.ts +1 -10
- package/build/services/engine/index.js +1 -48
- package/build/services/exporter/index.d.ts +0 -27
- package/build/services/exporter/index.js +0 -33
- package/build/services/hotmesh/index.d.ts +8 -4
- package/build/services/hotmesh/index.js +20 -19
- package/build/services/logger/index.js +0 -2
- package/build/services/mapper/index.d.ts +0 -14
- package/build/services/mapper/index.js +0 -14
- package/build/services/meshcall/index.d.ts +21 -0
- package/build/services/meshcall/index.js +202 -0
- package/build/services/meshcall/schemas/factory.d.ts +2 -0
- package/build/services/meshcall/schemas/factory.js +179 -0
- package/build/services/meshdata/index.d.ts +75 -0
- package/build/services/meshdata/index.js +541 -0
- package/build/services/meshflow/client.d.ts +18 -0
- package/build/services/{durable → meshflow}/client.js +9 -40
- package/build/services/{durable → meshflow}/connection.d.ts +2 -1
- package/build/services/{durable → meshflow}/connection.js +1 -0
- package/build/services/meshflow/exporter.d.ts +29 -0
- package/build/services/{durable → meshflow}/exporter.js +0 -29
- package/build/services/meshflow/handle.d.ts +22 -0
- package/build/services/{durable → meshflow}/handle.js +0 -46
- package/build/services/meshflow/index.d.ts +17 -0
- package/build/services/meshflow/index.js +23 -0
- package/build/services/meshflow/schemas/factory.d.ts +4 -0
- package/build/services/{durable → meshflow}/schemas/factory.js +2 -30
- package/build/services/meshflow/search.d.ts +23 -0
- package/build/services/{durable → meshflow}/search.js +0 -99
- package/build/services/{durable → meshflow}/worker.d.ts +3 -2
- package/build/services/{durable → meshflow}/worker.js +23 -39
- package/build/services/meshflow/workflow.d.ts +27 -0
- package/build/services/{durable → meshflow}/workflow.js +27 -169
- package/build/services/pipe/functions/date.d.ts +0 -7
- package/build/services/pipe/functions/date.js +0 -7
- package/build/services/pipe/functions/math.js +0 -2
- package/build/services/pipe/index.d.ts +0 -15
- package/build/services/pipe/index.js +2 -23
- package/build/services/quorum/index.d.ts +1 -7
- package/build/services/quorum/index.js +0 -21
- package/build/services/reporter/index.d.ts +0 -5
- package/build/services/reporter/index.js +0 -9
- package/build/services/router/index.d.ts +0 -9
- package/build/services/router/index.js +2 -30
- package/build/services/serializer/index.js +6 -23
- package/build/services/store/cache.d.ts +0 -19
- package/build/services/store/cache.js +0 -19
- package/build/services/store/clients/ioredis.d.ts +0 -6
- package/build/services/store/clients/ioredis.js +0 -7
- package/build/services/store/clients/redis.d.ts +0 -6
- package/build/services/store/clients/redis.js +0 -6
- package/build/services/store/index.d.ts +0 -55
- package/build/services/store/index.js +14 -87
- package/build/services/stream/clients/ioredis.js +1 -4
- package/build/services/task/index.d.ts +0 -9
- package/build/services/task/index.js +0 -31
- package/build/services/telemetry/index.d.ts +0 -7
- package/build/services/telemetry/index.js +1 -13
- package/build/services/worker/index.d.ts +1 -4
- package/build/services/worker/index.js +0 -6
- package/build/types/activity.d.ts +0 -81
- package/build/types/error.d.ts +5 -5
- package/build/types/exporter.d.ts +1 -14
- package/build/types/hotmesh.d.ts +4 -12
- package/build/types/hotmesh.js +0 -3
- package/build/types/index.d.ts +5 -3
- package/build/types/index.js +1 -1
- package/build/types/job.d.ts +1 -95
- package/build/types/meshcall.d.ts +54 -0
- package/build/types/meshdata.d.ts +59 -0
- package/build/types/meshdata.js +2 -0
- package/build/types/meshflow.d.ts +202 -0
- package/build/types/meshflow.js +2 -0
- package/build/types/pipe.d.ts +0 -65
- package/build/types/quorum.d.ts +0 -12
- package/build/types/redis.d.ts +0 -6
- package/build/types/stream.d.ts +0 -59
- package/build/types/stream.js +0 -4
- package/index.ts +12 -3
- package/package.json +18 -15
- package/typedoc.json +38 -0
- package/types/error.ts +5 -5
- package/types/exporter.ts +1 -1
- package/types/hotmesh.ts +3 -2
- package/types/index.ts +25 -7
- package/types/job.ts +19 -1
- package/types/meshcall.ts +192 -0
- package/types/meshdata.ts +273 -0
- package/types/{durable.ts → meshflow.ts} +33 -9
- package/build/services/durable/client.d.ts +0 -49
- package/build/services/durable/exporter.d.ts +0 -51
- package/build/services/durable/handle.d.ts +0 -58
- package/build/services/durable/index.d.ts +0 -19
- package/build/services/durable/index.js +0 -25
- package/build/services/durable/schemas/factory.d.ts +0 -33
- package/build/services/durable/search.d.ts +0 -120
- package/build/services/durable/workflow.d.ts +0 -143
- package/build/types/durable.d.ts +0 -467
- /package/build/types/{durable.js → meshcall.js} +0 -0
|
@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.MathHandler = void 0;
|
|
4
4
|
class MathHandler {
|
|
5
5
|
add(...operands) {
|
|
6
|
-
// @ts-ignore
|
|
7
6
|
return operands.reduce((a, b) => {
|
|
8
7
|
if (Array.isArray(b)) {
|
|
9
8
|
return a + this.add(...b);
|
|
@@ -38,7 +37,6 @@ class MathHandler {
|
|
|
38
37
|
if (operands.length === 0) {
|
|
39
38
|
throw new Error('At least one operand is required.');
|
|
40
39
|
}
|
|
41
|
-
// @ts-ignore
|
|
42
40
|
return operands.reduce((a, b) => {
|
|
43
41
|
if (Array.isArray(b)) {
|
|
44
42
|
return a * this.multiply(...b);
|
|
@@ -13,23 +13,8 @@ declare class Pipe {
|
|
|
13
13
|
static resolve(unresolved: {
|
|
14
14
|
[key: string]: unknown;
|
|
15
15
|
} | PipeItem, context: Partial<JobState>): any;
|
|
16
|
-
/**
|
|
17
|
-
* loop through each PipeItem row in this Pipe, resolving and transforming line by line
|
|
18
|
-
* @returns {any} the result of the pipe
|
|
19
|
-
*/
|
|
20
16
|
process(resolved?: unknown[] | null): any;
|
|
21
17
|
cloneUnknown<T>(value: T): T;
|
|
22
|
-
/**
|
|
23
|
-
* Transforms iterable `input` into a single value. Vars $output, $item, $key
|
|
24
|
-
* and $input are available. The final statement in the iterator (the reduction)
|
|
25
|
-
* is assumed to be the return value. A default $output object may be provided
|
|
26
|
-
* to the iterator by placing the the second cell of the preceding row. Otherwise,
|
|
27
|
-
* construct the object during first run and ensure it is the first cell of the
|
|
28
|
-
* last row of the iterator, so it is returned as the $output for the next cycle
|
|
29
|
-
* @param {unknown[]} input
|
|
30
|
-
* @returns {unknown}
|
|
31
|
-
* @private
|
|
32
|
-
*/
|
|
33
18
|
reduce(input: Array<unknown[]>): unknown;
|
|
34
19
|
private processRow;
|
|
35
20
|
static resolveFunction(functionName: string): any;
|
|
@@ -33,16 +33,12 @@ class Pipe {
|
|
|
33
33
|
}
|
|
34
34
|
return pipe.process();
|
|
35
35
|
}
|
|
36
|
-
/**
|
|
37
|
-
* loop through each PipeItem row in this Pipe, resolving and transforming line by line
|
|
38
|
-
* @returns {any} the result of the pipe
|
|
39
|
-
*/
|
|
40
36
|
process(resolved = null) {
|
|
41
37
|
let index = 0;
|
|
42
38
|
if (!(resolved ||
|
|
43
39
|
this.isPipeType(this.rules[0]) ||
|
|
44
40
|
this.isreduceType(this.rules[0]))) {
|
|
45
|
-
resolved = this.processCells(this.rules[0]);
|
|
41
|
+
resolved = this.processCells(this.rules[0]);
|
|
46
42
|
index = 1;
|
|
47
43
|
}
|
|
48
44
|
const len = this.rules.length;
|
|
@@ -73,17 +69,6 @@ class Pipe {
|
|
|
73
69
|
}
|
|
74
70
|
return clonedObj;
|
|
75
71
|
}
|
|
76
|
-
/**
|
|
77
|
-
* Transforms iterable `input` into a single value. Vars $output, $item, $key
|
|
78
|
-
* and $input are available. The final statement in the iterator (the reduction)
|
|
79
|
-
* is assumed to be the return value. A default $output object may be provided
|
|
80
|
-
* to the iterator by placing the the second cell of the preceding row. Otherwise,
|
|
81
|
-
* construct the object during first run and ensure it is the first cell of the
|
|
82
|
-
* last row of the iterator, so it is returned as the $output for the next cycle
|
|
83
|
-
* @param {unknown[]} input
|
|
84
|
-
* @returns {unknown}
|
|
85
|
-
* @private
|
|
86
|
-
*/
|
|
87
72
|
reduce(input) {
|
|
88
73
|
let resolved = this.cloneUnknown(input[1] ?? null);
|
|
89
74
|
if (Array.isArray(input[0])) {
|
|
@@ -116,30 +101,25 @@ class Pipe {
|
|
|
116
101
|
}
|
|
117
102
|
processRow(currentRow, resolvedPriorRow, subPipeQueue) {
|
|
118
103
|
if (resolvedPriorRow && this.isreduceType(currentRow)) {
|
|
119
|
-
//reduce the resolvedPriorRow and return the output from the reducer
|
|
120
104
|
const subPipe = new Pipe(currentRow['@reduce'], this.jobData);
|
|
121
105
|
const reduced = subPipe.reduce(resolvedPriorRow);
|
|
122
106
|
return reduced;
|
|
123
107
|
}
|
|
124
108
|
else if (this.isPipeType(currentRow)) {
|
|
125
|
-
//process subPipe and push to subPipeQueue; echo resolvedPriorRow
|
|
126
109
|
const subPipe = new Pipe(currentRow['@pipe'], this.jobData, this.context);
|
|
127
110
|
subPipeQueue.push(subPipe.process());
|
|
128
111
|
return resolvedPriorRow;
|
|
129
112
|
}
|
|
130
113
|
else {
|
|
131
|
-
//pivot the subPipeQueue into the arguments array (resolvedPriorRow)
|
|
132
114
|
if (subPipeQueue.length > 0) {
|
|
133
115
|
resolvedPriorRow = [...subPipeQueue];
|
|
134
116
|
subPipeQueue.length = 0;
|
|
135
117
|
}
|
|
136
118
|
if (!resolvedPriorRow) {
|
|
137
|
-
//if no prior row, use current row as prior row
|
|
138
119
|
return [].concat(this.processCells(Array.isArray(currentRow) ? [...currentRow] : []));
|
|
139
120
|
}
|
|
140
121
|
else {
|
|
141
|
-
const [functionName, ...params] = currentRow;
|
|
142
|
-
//use resolved values from prior row (n - 1) as input params to cell 1 function
|
|
122
|
+
const [functionName, ...params] = currentRow;
|
|
143
123
|
let resolvedValue;
|
|
144
124
|
if (this.isContextVariable(functionName)) {
|
|
145
125
|
resolvedValue = this.resolveContextValue(functionName);
|
|
@@ -147,7 +127,6 @@ class Pipe {
|
|
|
147
127
|
else {
|
|
148
128
|
resolvedValue = Pipe.resolveFunction(functionName)(...resolvedPriorRow);
|
|
149
129
|
}
|
|
150
|
-
//resolve remaining cells in row and return concatenated with resolvedValue
|
|
151
130
|
return [resolvedValue].concat(this.processCells([...params]));
|
|
152
131
|
}
|
|
153
132
|
}
|
|
@@ -21,6 +21,7 @@ declare class QuorumService {
|
|
|
21
21
|
quorum: number | null;
|
|
22
22
|
callbacks: QuorumMessageCallback[];
|
|
23
23
|
rollCallInterval: NodeJS.Timeout;
|
|
24
|
+
constructor();
|
|
24
25
|
static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<QuorumService>;
|
|
25
26
|
verifyQuorumFields(config: HotMeshConfig): void;
|
|
26
27
|
initStoreChannel(store: RedisClient): Promise<void>;
|
|
@@ -28,10 +29,6 @@ declare class QuorumService {
|
|
|
28
29
|
subscriptionHandler(): SubscriptionCallback;
|
|
29
30
|
sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
|
|
30
31
|
requestQuorum(delay?: number, details?: boolean): Promise<number>;
|
|
31
|
-
/**
|
|
32
|
-
* A quorum-wide command to broadcaset system details.
|
|
33
|
-
*
|
|
34
|
-
*/
|
|
35
32
|
doRollCall(message: RollCallMessage): Promise<void>;
|
|
36
33
|
cancelRollCall(): void;
|
|
37
34
|
stop(): void;
|
|
@@ -39,9 +36,6 @@ declare class QuorumService {
|
|
|
39
36
|
sub(callback: QuorumMessageCallback): Promise<void>;
|
|
40
37
|
unsub(callback: QuorumMessageCallback): Promise<void>;
|
|
41
38
|
rollCall(delay?: number): Promise<QuorumProfile[]>;
|
|
42
|
-
/**
|
|
43
|
-
* request a quorum; if successful activate the app version
|
|
44
|
-
*/
|
|
45
39
|
activate(version: string, delay?: number, count?: number): Promise<boolean>;
|
|
46
40
|
}
|
|
47
41
|
export { QuorumService };
|
|
@@ -26,12 +26,9 @@ class QuorumService {
|
|
|
26
26
|
instance.guid = guid;
|
|
27
27
|
instance.logger = logger;
|
|
28
28
|
instance.engine = engine;
|
|
29
|
-
//note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
|
|
30
29
|
await instance.initStoreChannel(config.engine.store);
|
|
31
30
|
await instance.initSubChannel(config.engine.sub);
|
|
32
|
-
//general quorum subscription
|
|
33
31
|
await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
|
|
34
|
-
//app-specific quorum subscription (used for pubsub one-time request/response)
|
|
35
32
|
await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
|
|
36
33
|
instance.engine.processWebHooks();
|
|
37
34
|
instance.engine.processTimeHooks();
|
|
@@ -93,7 +90,6 @@ class QuorumService {
|
|
|
93
90
|
else if (message.type === 'rollcall') {
|
|
94
91
|
self.doRollCall(message);
|
|
95
92
|
}
|
|
96
|
-
//if there are any callbacks, call them
|
|
97
93
|
if (self.callbacks.length > 0) {
|
|
98
94
|
self.callbacks.forEach((cb) => cb(topic, message));
|
|
99
95
|
}
|
|
@@ -138,10 +134,6 @@ class QuorumService {
|
|
|
138
134
|
await (0, utils_1.sleepFor)(delay);
|
|
139
135
|
return quorum;
|
|
140
136
|
}
|
|
141
|
-
/**
|
|
142
|
-
* A quorum-wide command to broadcaset system details.
|
|
143
|
-
*
|
|
144
|
-
*/
|
|
145
137
|
async doRollCall(message) {
|
|
146
138
|
let iteration = 0;
|
|
147
139
|
const max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
|
|
@@ -168,22 +160,15 @@ class QuorumService {
|
|
|
168
160
|
stop() {
|
|
169
161
|
this.cancelRollCall();
|
|
170
162
|
}
|
|
171
|
-
// ************* PUB/SUB METHODS *************
|
|
172
|
-
//publish a message to the quorum
|
|
173
163
|
async pub(quorumMessage) {
|
|
174
164
|
return await this.store.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
|
|
175
165
|
}
|
|
176
|
-
//subscribe user to quorum messages
|
|
177
166
|
async sub(callback) {
|
|
178
|
-
//the quorum is always subscribed to the `quorum` topic; just register the fn
|
|
179
167
|
this.callbacks.push(callback);
|
|
180
168
|
}
|
|
181
|
-
//unsubscribe user from quorum messages
|
|
182
169
|
async unsub(callback) {
|
|
183
|
-
//the quorum is always subscribed to the `quorum` topic; just unregister the fn
|
|
184
170
|
this.callbacks = this.callbacks.filter((cb) => cb !== callback);
|
|
185
171
|
}
|
|
186
|
-
// ************* COMPILER METHODS *************
|
|
187
172
|
async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
|
|
188
173
|
await this.requestQuorum(delay, true);
|
|
189
174
|
const targetStreams = [];
|
|
@@ -205,14 +190,10 @@ class QuorumService {
|
|
|
205
190
|
});
|
|
206
191
|
return this.profiles;
|
|
207
192
|
}
|
|
208
|
-
/**
|
|
209
|
-
* request a quorum; if successful activate the app version
|
|
210
|
-
*/
|
|
211
193
|
async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
|
|
212
194
|
version = version.toString();
|
|
213
195
|
const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
|
|
214
196
|
if (!canActivate) {
|
|
215
|
-
//another engine is already activating the app version
|
|
216
197
|
this.logger.debug('quorum-activation-awaiting', { version });
|
|
217
198
|
await (0, utils_1.sleepFor)(delay * 6);
|
|
218
199
|
const app = await this.store.getApp(this.appId, true);
|
|
@@ -228,7 +209,6 @@ class QuorumService {
|
|
|
228
209
|
this.store.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
|
|
229
210
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
230
211
|
await this.store.releaseScoutRole('activate');
|
|
231
|
-
//confirm we received the activation message
|
|
232
212
|
if (this.engine.untilVersion === version) {
|
|
233
213
|
this.logger.info('quorum-activation-succeeded', { version });
|
|
234
214
|
const { id } = config;
|
|
@@ -244,7 +224,6 @@ class QuorumService {
|
|
|
244
224
|
this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
|
|
245
225
|
this.store.releaseScoutRole('activate');
|
|
246
226
|
if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
|
|
247
|
-
//increase the delay (give the quorum time to respond) and try again
|
|
248
227
|
return await this.activate(version, delay * 2, count + 1);
|
|
249
228
|
}
|
|
250
229
|
throw new Error(`Quorum not reached. Version ${version} not activated.`);
|
|
@@ -28,11 +28,6 @@ declare class ReporterService {
|
|
|
28
28
|
getTargetForTime(key: string): string;
|
|
29
29
|
getWorkItems(options: GetStatsOptions, facets: string[]): Promise<string[]>;
|
|
30
30
|
private buildWorkerLists;
|
|
31
|
-
/**
|
|
32
|
-
* called by `trigger` activity to generate the stats that should
|
|
33
|
-
* be saved to the database. doesn't actually save the stats, but
|
|
34
|
-
* just generates the info that should be saved
|
|
35
|
-
*/
|
|
36
31
|
resolveTriggerStatistics({ stats: statsConfig }: TriggerActivity, context: JobState): StatsType;
|
|
37
32
|
isGeneralMetric(metric: string): boolean;
|
|
38
33
|
isMedianMetric(metric: string): boolean;
|
|
@@ -28,11 +28,9 @@ class ReporterService {
|
|
|
28
28
|
}
|
|
29
29
|
generateDateTimeSets(granularity, range, end, start) {
|
|
30
30
|
if (granularity === 'infinity') {
|
|
31
|
-
//if granularity is infinity, it means a date/time sequence/slice is not used to further segment the statistics
|
|
32
31
|
return ['0'];
|
|
33
32
|
}
|
|
34
33
|
if (!range) {
|
|
35
|
-
//pluck just a single value when no range provided
|
|
36
34
|
range = '0m';
|
|
37
35
|
}
|
|
38
36
|
const granularitiesInMinutes = {
|
|
@@ -50,7 +48,6 @@ class ReporterService {
|
|
|
50
48
|
if (rangeMinutes === null) {
|
|
51
49
|
throw new Error('Invalid range value.');
|
|
52
50
|
}
|
|
53
|
-
// If start is provided, use it. Otherwise, calculate it from the end time and range.
|
|
54
51
|
let startTime;
|
|
55
52
|
let endTime;
|
|
56
53
|
if (start) {
|
|
@@ -61,7 +58,6 @@ class ReporterService {
|
|
|
61
58
|
endTime = end === 'NOW' ? new Date() : new Date(end);
|
|
62
59
|
startTime = new Date(endTime.getTime() - rangeMinutes * 60 * 1000);
|
|
63
60
|
}
|
|
64
|
-
// Round the start time to the nearest granularity unit
|
|
65
61
|
startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) *
|
|
66
62
|
granularityMinutes);
|
|
67
63
|
const dateTimeSets = [];
|
|
@@ -273,11 +269,6 @@ class ReporterService {
|
|
|
273
269
|
}
|
|
274
270
|
return workerLists;
|
|
275
271
|
}
|
|
276
|
-
/**
|
|
277
|
-
* called by `trigger` activity to generate the stats that should
|
|
278
|
-
* be saved to the database. doesn't actually save the stats, but
|
|
279
|
-
* just generates the info that should be saved
|
|
280
|
-
*/
|
|
281
272
|
resolveTriggerStatistics({ stats: statsConfig }, context) {
|
|
282
273
|
const stats = {
|
|
283
274
|
general: [],
|
|
@@ -30,15 +30,6 @@ declare class Router {
|
|
|
30
30
|
private resetThrottleState;
|
|
31
31
|
createGroup(stream: string, group: string): Promise<void>;
|
|
32
32
|
publishMessage(topic: string, streamData: StreamData | StreamDataResponse, multi?: RedisMulti): Promise<string | RedisMulti>;
|
|
33
|
-
/**
|
|
34
|
-
* An adjustable throttle that will interrupt a sleeping
|
|
35
|
-
* router if the throttle is reduced and the sleep time
|
|
36
|
-
* has elapsed. If the throttle is increased, or if
|
|
37
|
-
* the sleep time has not elapsed, the router will continue
|
|
38
|
-
* to sleep until the new termination point. This
|
|
39
|
-
* allows for dynamic, elastic throttling with smooth
|
|
40
|
-
* acceleration and deceleration.
|
|
41
|
-
*/
|
|
42
33
|
customSleep(): Promise<void>;
|
|
43
34
|
consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
|
|
44
35
|
isStreamMessage(result: any): boolean;
|
|
@@ -51,22 +51,13 @@ class Router {
|
|
|
51
51
|
});
|
|
52
52
|
return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
|
|
53
53
|
}
|
|
54
|
-
/**
|
|
55
|
-
* An adjustable throttle that will interrupt a sleeping
|
|
56
|
-
* router if the throttle is reduced and the sleep time
|
|
57
|
-
* has elapsed. If the throttle is increased, or if
|
|
58
|
-
* the sleep time has not elapsed, the router will continue
|
|
59
|
-
* to sleep until the new termination point. This
|
|
60
|
-
* allows for dynamic, elastic throttling with smooth
|
|
61
|
-
* acceleration and deceleration.
|
|
62
|
-
*/
|
|
63
54
|
async customSleep() {
|
|
64
55
|
if (this.throttle === 0)
|
|
65
56
|
return;
|
|
66
57
|
if (this.isSleeping)
|
|
67
58
|
return;
|
|
68
59
|
this.isSleeping = true;
|
|
69
|
-
const startTime = Date.now();
|
|
60
|
+
const startTime = Date.now();
|
|
70
61
|
await new Promise(async (outerResolve) => {
|
|
71
62
|
this.sleepPromiseResolve = outerResolve;
|
|
72
63
|
let elapsedTime = Date.now() - startTime;
|
|
@@ -97,7 +88,6 @@ class Router {
|
|
|
97
88
|
return;
|
|
98
89
|
}
|
|
99
90
|
try {
|
|
100
|
-
//randomizer that asymptotes at 150% of `HMSH_BLOCK_TIME_MS`
|
|
101
91
|
const streamDuration = enums_1.HMSH_BLOCK_TIME_MS + Math.round(enums_1.HMSH_BLOCK_TIME_MS * Math.random());
|
|
102
92
|
const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', streamDuration, 'STREAMS', stream, '>');
|
|
103
93
|
if (this.isStopped(group, consumer, stream)) {
|
|
@@ -113,7 +103,6 @@ class Router {
|
|
|
113
103
|
await this.consumeOne(stream, group, id, message, callback);
|
|
114
104
|
}
|
|
115
105
|
}
|
|
116
|
-
// Check for pending messages (note: Redis 6.2 simplifies)
|
|
117
106
|
const now = Date.now();
|
|
118
107
|
if (now - lastCheckedPendingMessagesAt > this.reclaimDelay) {
|
|
119
108
|
lastCheckedPendingMessagesAt = now;
|
|
@@ -213,7 +202,6 @@ class Router {
|
|
|
213
202
|
await (0, utils_1.sleepFor)(timeout);
|
|
214
203
|
return (await this.publishMessage(input.metadata.topic, {
|
|
215
204
|
data: input.data,
|
|
216
|
-
//note: retain guid (this is a retry attempt)
|
|
217
205
|
metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
|
|
218
206
|
policies: input.policies,
|
|
219
207
|
}));
|
|
@@ -233,16 +221,12 @@ class Router {
|
|
|
233
221
|
}
|
|
234
222
|
}
|
|
235
223
|
shouldRetry(input, output) {
|
|
236
|
-
//const isUnhandledEngineError = output.code === 500;
|
|
237
224
|
const policies = input.policies?.retry;
|
|
238
225
|
const errorCode = output.code.toString();
|
|
239
226
|
const policy = policies?.[errorCode];
|
|
240
227
|
const maxRetries = policy?.[0];
|
|
241
228
|
const tryCount = Math.min(input.metadata.try || 0, enums_1.HMSH_MAX_RETRIES);
|
|
242
|
-
//only possible values for maxRetries are 1, 2, 3
|
|
243
|
-
//only possible values for tryCount are 0, 1, 2
|
|
244
229
|
if (maxRetries > tryCount) {
|
|
245
|
-
// 10ms, 100ms, or 1000ms delays between system retries
|
|
246
230
|
return [true, Math.pow(10, tryCount + 1)];
|
|
247
231
|
}
|
|
248
232
|
return [false, 0];
|
|
@@ -278,7 +262,6 @@ class Router {
|
|
|
278
262
|
code,
|
|
279
263
|
data,
|
|
280
264
|
};
|
|
281
|
-
//send unacknowleded errors to the engine (it has no topic)
|
|
282
265
|
delete output.metadata.topic;
|
|
283
266
|
return output;
|
|
284
267
|
}
|
|
@@ -330,7 +313,6 @@ class Router {
|
|
|
330
313
|
}
|
|
331
314
|
const wasDecreased = delayInMillis < this.throttle;
|
|
332
315
|
this.throttle = delayInMillis;
|
|
333
|
-
// If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
|
|
334
316
|
if (wasDecreased) {
|
|
335
317
|
if (this.sleepTimout) {
|
|
336
318
|
clearTimeout(this.sleepTimout);
|
|
@@ -342,7 +324,7 @@ class Router {
|
|
|
342
324
|
}
|
|
343
325
|
async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = enums_1.HMSH_XPENDING_COUNT) {
|
|
344
326
|
let pendingMessages = [];
|
|
345
|
-
const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit);
|
|
327
|
+
const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit);
|
|
346
328
|
for (const pendingMessageInfo of pendingMessagesInfo) {
|
|
347
329
|
if (Array.isArray(pendingMessageInfo)) {
|
|
348
330
|
const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
|
|
@@ -362,13 +344,6 @@ class Router {
|
|
|
362
344
|
return pendingMessages;
|
|
363
345
|
}
|
|
364
346
|
async expireUnacknowledged(reclaimedMessage, stream, group, consumer, id, count) {
|
|
365
|
-
//The stream activity was not processed within established limits. Possibilities Include:
|
|
366
|
-
// 1) user error: the workers were not properly configured and are timing out
|
|
367
|
-
// 2a) system error: JSON is corrupt
|
|
368
|
-
// i) unwitting actor
|
|
369
|
-
// ii) corrupt hardware/network/transport/etc
|
|
370
|
-
// 3b) system error: Redis unable to accept `xadd` request
|
|
371
|
-
// 4c) system error: Redis unable to accept `xdel`/`xack` request
|
|
372
347
|
this.logger.error('stream-message-max-delivery-count-exceeded', {
|
|
373
348
|
id,
|
|
374
349
|
stream,
|
|
@@ -378,7 +353,6 @@ class Router {
|
|
|
378
353
|
count,
|
|
379
354
|
});
|
|
380
355
|
const streamData = reclaimedMessage[0]?.[1]?.[1];
|
|
381
|
-
//fatal risk point 1 of 3): json is corrupt
|
|
382
356
|
const [err, input] = this.parseStreamData(streamData);
|
|
383
357
|
if (err) {
|
|
384
358
|
return this.logger.error('expire-unacknowledged-parse-fatal-error', {
|
|
@@ -397,11 +371,9 @@ class Router {
|
|
|
397
371
|
telemetry = new telemetry_1.TelemetryService(this.appId);
|
|
398
372
|
telemetry.startStreamSpan(input, stream_1.StreamRole.SYSTEM);
|
|
399
373
|
telemetry.setStreamError(`Stream Message Max Delivery Count Exceeded`);
|
|
400
|
-
//fatal risk point 2 of 3): unable to publish error message (to notify the parent job)
|
|
401
374
|
const output = this.structureUnacknowledgedError(input);
|
|
402
375
|
messageId = await this.publishResponse(input, output);
|
|
403
376
|
telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
|
|
404
|
-
//fatal risk point 3 of 3): unable to ack and delete stream message
|
|
405
377
|
await this.ackAndDelete(stream, group, id);
|
|
406
378
|
}
|
|
407
379
|
catch (err) {
|
|
@@ -69,7 +69,6 @@ class SerializerService {
|
|
|
69
69
|
return this.dIds[activityId];
|
|
70
70
|
}
|
|
71
71
|
else if ('$ADJACENT' in this.dIds) {
|
|
72
|
-
//else=> pre-authorizing adjacent activity entry
|
|
73
72
|
return this.dIds['$ADJACENT'];
|
|
74
73
|
}
|
|
75
74
|
return ',0';
|
|
@@ -140,7 +139,6 @@ class SerializerService {
|
|
|
140
139
|
result[shortDimensionalKey] = source[key];
|
|
141
140
|
}
|
|
142
141
|
else if (!(key in result) && this.isLiteralKeyType(key)) {
|
|
143
|
-
//mark (-) and search (_)
|
|
144
142
|
result[key] = source[key];
|
|
145
143
|
}
|
|
146
144
|
}
|
|
@@ -164,7 +162,6 @@ class SerializerService {
|
|
|
164
162
|
const inflateWithMap = (abbreviationMap, id) => {
|
|
165
163
|
const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
|
|
166
164
|
for (const key in result) {
|
|
167
|
-
//strip dimensional index from key
|
|
168
165
|
const shortKey = key.split(',')[0];
|
|
169
166
|
const longKey = reversedAbbreviationMap.get(shortKey);
|
|
170
167
|
if (longKey) {
|
|
@@ -181,36 +178,22 @@ class SerializerService {
|
|
|
181
178
|
}
|
|
182
179
|
return result;
|
|
183
180
|
}
|
|
184
|
-
//stringify: convert a multi-dimensional document to a 2-d hash
|
|
185
181
|
stringify(document) {
|
|
186
182
|
const result = {};
|
|
187
183
|
for (const key in document) {
|
|
188
184
|
const value = SerializerService.toString(document[key]);
|
|
189
185
|
if (value) {
|
|
190
|
-
// if (/^:*[a-zA-Z]{2}$/.test(value)) {
|
|
191
|
-
// value = ':' + value;
|
|
192
|
-
// } else if (this.symValReverseMaps.has(value)) {
|
|
193
|
-
// value = this.symValReverseMaps.get(value);
|
|
194
|
-
// }
|
|
195
186
|
result[key] = value;
|
|
196
187
|
}
|
|
197
188
|
}
|
|
198
189
|
return result;
|
|
199
190
|
}
|
|
200
|
-
//parse: convert a 2-d hash to a multi-dimensional document
|
|
201
191
|
parse(document) {
|
|
202
192
|
const result = {};
|
|
203
193
|
for (const [key, value] of Object.entries(document)) {
|
|
204
194
|
if (value === undefined || value === null)
|
|
205
195
|
continue;
|
|
206
|
-
// if (/^:+[a-zA-Z]{2}$/.test(value)) {
|
|
207
|
-
// result[key] = value.slice(1);
|
|
208
|
-
// } else {
|
|
209
|
-
// if (value?.length === 2 && this.symValMaps.has(value)) {
|
|
210
|
-
// value = this.symValMaps.get(value);
|
|
211
|
-
// }
|
|
212
196
|
result[key] = SerializerService.fromString(value);
|
|
213
|
-
// }
|
|
214
197
|
}
|
|
215
198
|
return result;
|
|
216
199
|
}
|
|
@@ -243,20 +226,20 @@ class SerializerService {
|
|
|
243
226
|
const prefix = value.slice(0, 2);
|
|
244
227
|
const rest = value.slice(2);
|
|
245
228
|
switch (prefix) {
|
|
246
|
-
case '/t':
|
|
229
|
+
case '/t':
|
|
247
230
|
return true;
|
|
248
|
-
case '/f':
|
|
231
|
+
case '/f':
|
|
249
232
|
return false;
|
|
250
|
-
case '/d':
|
|
233
|
+
case '/d':
|
|
251
234
|
return Number(rest);
|
|
252
|
-
case '/n':
|
|
235
|
+
case '/n':
|
|
253
236
|
return null;
|
|
254
|
-
case '/s':
|
|
237
|
+
case '/s':
|
|
255
238
|
if (dateReg.exec(rest)) {
|
|
256
239
|
return new Date(JSON.parse(rest));
|
|
257
240
|
}
|
|
258
241
|
return JSON.parse(rest);
|
|
259
|
-
default:
|
|
242
|
+
default:
|
|
260
243
|
return value;
|
|
261
244
|
}
|
|
262
245
|
}
|
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The Cache is a key/value store and used to store commonly accessed Redis metadata
|
|
3
|
-
* (mainly the execution rules for the app) to save time accessing them as they
|
|
4
|
-
* are immutable per verison. Rules are only ejected when a new version
|
|
5
|
-
* (a new distributed executable) is deployed to the quorum
|
|
6
|
-
* and the cache is invalidated/cleared of the prior version.
|
|
7
|
-
*/
|
|
8
1
|
import { ActivityType } from '../../types/activity';
|
|
9
2
|
import { HookRule } from '../../types/hook';
|
|
10
3
|
import { HotMeshApp, HotMeshSettings } from '../../types/hotmesh';
|
|
@@ -21,19 +14,7 @@ declare class Cache {
|
|
|
21
14
|
transitions: Record<string, Record<string, unknown>>;
|
|
22
15
|
hookRules: Record<string, Record<string, HookRule[]>>;
|
|
23
16
|
workItems: Record<string, string>;
|
|
24
|
-
/**
|
|
25
|
-
* The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
|
|
26
|
-
* @param settings
|
|
27
|
-
* @param apps
|
|
28
|
-
* @param schemas
|
|
29
|
-
* @param subscriptions
|
|
30
|
-
* @param transitions
|
|
31
|
-
* @param hookRules
|
|
32
|
-
*/
|
|
33
17
|
constructor(appId: string, settings: HotMeshSettings, apps?: Record<string, HotMeshApp>, schemas?: Record<string, ActivityType>, subscriptions?: Record<string, Record<string, string>>, symbols?: Record<string, Symbols>, symvals?: Record<string, Symbols>, transitions?: Record<string, Record<string, unknown>>, hookRules?: Record<string, Record<string, HookRule[]>>, workItems?: Record<string, string>);
|
|
34
|
-
/**
|
|
35
|
-
* invalidate the cache; settings are not invalidated!
|
|
36
|
-
*/
|
|
37
18
|
invalidate(): void;
|
|
38
19
|
getSettings(): HotMeshSettings;
|
|
39
20
|
setSettings(settings: HotMeshSettings): void;
|
|
@@ -1,23 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* The Cache is a key/value store and used to store commonly accessed Redis metadata
|
|
4
|
-
* (mainly the execution rules for the app) to save time accessing them as they
|
|
5
|
-
* are immutable per verison. Rules are only ejected when a new version
|
|
6
|
-
* (a new distributed executable) is deployed to the quorum
|
|
7
|
-
* and the cache is invalidated/cleared of the prior version.
|
|
8
|
-
*/
|
|
9
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
3
|
exports.Cache = void 0;
|
|
11
4
|
class Cache {
|
|
12
|
-
/**
|
|
13
|
-
* The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
|
|
14
|
-
* @param settings
|
|
15
|
-
* @param apps
|
|
16
|
-
* @param schemas
|
|
17
|
-
* @param subscriptions
|
|
18
|
-
* @param transitions
|
|
19
|
-
* @param hookRules
|
|
20
|
-
*/
|
|
21
5
|
constructor(appId, settings, apps = {}, schemas = {}, subscriptions = {}, symbols = {}, symvals = {}, transitions = {}, hookRules = {}, workItems = {}) {
|
|
22
6
|
this.appId = appId;
|
|
23
7
|
this.settings = settings;
|
|
@@ -30,9 +14,6 @@ class Cache {
|
|
|
30
14
|
this.hookRules = hookRules;
|
|
31
15
|
this.workItems = workItems;
|
|
32
16
|
}
|
|
33
|
-
/**
|
|
34
|
-
* invalidate the cache; settings are not invalidated!
|
|
35
|
-
*/
|
|
36
17
|
invalidate() {
|
|
37
18
|
this.apps = {};
|
|
38
19
|
this.schemas = {};
|
|
@@ -13,12 +13,6 @@ declare class IORedisStoreService extends StoreService<RedisClientType, RedisMul
|
|
|
13
13
|
logger: ILogger;
|
|
14
14
|
serializer: Serializer;
|
|
15
15
|
constructor(redisClient: RedisClientType);
|
|
16
|
-
/**
|
|
17
|
-
* When in cluster mode, the getMulti wrapper only
|
|
18
|
-
* sends commands to the same node/shard if they share a key.
|
|
19
|
-
* All other commands are sent simultaneouslyusing Promise.all
|
|
20
|
-
* and are then collated
|
|
21
|
-
*/
|
|
22
16
|
getMulti(): RedisMultiType;
|
|
23
17
|
exec(...args: any[]): Promise<string | string[] | string[][]>;
|
|
24
18
|
hGetAllResult(result: any): any;
|
|
@@ -8,12 +8,6 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
8
8
|
constructor(redisClient) {
|
|
9
9
|
super(redisClient);
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
12
|
-
* When in cluster mode, the getMulti wrapper only
|
|
13
|
-
* sends commands to the same node/shard if they share a key.
|
|
14
|
-
* All other commands are sent simultaneouslyusing Promise.all
|
|
15
|
-
* and are then collated
|
|
16
|
-
*/
|
|
17
11
|
getMulti() {
|
|
18
12
|
const my = this;
|
|
19
13
|
if (enums_1.HMSH_IS_CLUSTER) {
|
|
@@ -129,7 +123,6 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
129
123
|
return response;
|
|
130
124
|
}
|
|
131
125
|
hGetAllResult(result) {
|
|
132
|
-
//ioredis response signature is [null, {}] or [null, null]
|
|
133
126
|
return result[1];
|
|
134
127
|
}
|
|
135
128
|
async addTaskQueues(keys) {
|
|
@@ -14,12 +14,6 @@ declare class RedisStoreService extends StoreService<RedisClientType, RedisMulti
|
|
|
14
14
|
serializer: Serializer;
|
|
15
15
|
commands: Record<string, string>;
|
|
16
16
|
constructor(redisClient: RedisClientType);
|
|
17
|
-
/**
|
|
18
|
-
* When in cluster mode, the getMulti wrapper only
|
|
19
|
-
* sends commands to the same node/shard if they share a key.
|
|
20
|
-
* All other commands are sent simultaneouslyusing Promise.all
|
|
21
|
-
* and are then collated
|
|
22
|
-
*/
|
|
23
17
|
getMulti(): RedisMultiType;
|
|
24
18
|
exec(...args: any[]): Promise<string | string[] | string[][]>;
|
|
25
19
|
publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean>;
|
|
@@ -37,12 +37,6 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
37
37
|
xlen: 'XLEN',
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
/**
|
|
41
|
-
* When in cluster mode, the getMulti wrapper only
|
|
42
|
-
* sends commands to the same node/shard if they share a key.
|
|
43
|
-
* All other commands are sent simultaneouslyusing Promise.all
|
|
44
|
-
* and are then collated
|
|
45
|
-
*/
|
|
46
40
|
getMulti() {
|
|
47
41
|
const my = this;
|
|
48
42
|
if (enums_1.HMSH_IS_CLUSTER) {
|