@hotmeshio/hotmesh 0.0.59 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/modules/enums.d.ts +2 -1
- package/build/modules/enums.js +13 -3
- package/build/modules/errors.d.ts +3 -3
- package/build/modules/errors.js +8 -8
- package/build/modules/key.d.ts +1 -1
- package/build/modules/key.js +3 -3
- package/build/modules/utils.d.ts +6 -6
- package/build/modules/utils.js +22 -14
- package/build/package.json +41 -38
- package/build/services/activities/activity.js +37 -20
- package/build/services/activities/await.d.ts +1 -1
- package/build/services/activities/await.js +15 -7
- package/build/services/activities/cycle.d.ts +1 -1
- package/build/services/activities/cycle.js +16 -8
- package/build/services/activities/hook.d.ts +1 -1
- package/build/services/activities/hook.js +8 -4
- package/build/services/activities/interrupt.d.ts +1 -1
- package/build/services/activities/interrupt.js +14 -6
- package/build/services/activities/signal.d.ts +1 -1
- package/build/services/activities/signal.js +12 -4
- package/build/services/activities/trigger.d.ts +1 -1
- package/build/services/activities/trigger.js +19 -12
- package/build/services/activities/worker.d.ts +1 -1
- package/build/services/activities/worker.js +15 -7
- package/build/services/collator/index.js +12 -12
- package/build/services/compiler/deployer.js +17 -12
- package/build/services/compiler/index.js +4 -4
- package/build/services/compiler/validator.d.ts +3 -3
- package/build/services/compiler/validator.js +12 -3
- package/build/services/durable/client.d.ts +1 -1
- package/build/services/durable/client.js +18 -12
- package/build/services/durable/connection.d.ts +1 -1
- package/build/services/durable/exporter.d.ts +1 -1
- package/build/services/durable/exporter.js +3 -4
- package/build/services/durable/handle.d.ts +1 -1
- package/build/services/durable/handle.js +4 -1
- package/build/services/durable/index.d.ts +1 -1
- package/build/services/durable/index.js +2 -2
- package/build/services/durable/schemas/factory.d.ts +1 -1
- package/build/services/durable/search.js +19 -11
- package/build/services/durable/worker.js +50 -30
- package/build/services/durable/workflow.d.ts +5 -5
- package/build/services/durable/workflow.js +34 -18
- package/build/services/engine/index.js +33 -26
- package/build/services/exporter/index.d.ts +1 -1
- package/build/services/exporter/index.js +3 -3
- package/build/services/hotmesh/index.js +1 -1
- package/build/services/logger/index.js +1 -1
- package/build/services/mapper/index.js +3 -1
- package/build/services/pipe/functions/date.js +1 -1
- package/build/services/pipe/index.js +37 -10
- package/build/services/quorum/index.js +14 -11
- package/build/services/reporter/index.js +15 -12
- package/build/services/router/index.d.ts +2 -2
- package/build/services/router/index.js +73 -23
- package/build/services/serializer/index.js +48 -26
- package/build/services/store/cache.d.ts +5 -5
- package/build/services/store/cache.js +2 -2
- package/build/services/store/clients/ioredis.js +3 -3
- package/build/services/store/clients/redis.js +24 -4
- package/build/services/store/index.d.ts +9 -3
- package/build/services/store/index.js +122 -60
- package/build/services/stream/clients/ioredis.js +4 -4
- package/build/services/stream/clients/redis.js +31 -4
- package/build/services/task/index.js +8 -11
- package/build/services/telemetry/index.js +21 -14
- package/build/services/worker/index.d.ts +6 -6
- package/build/services/worker/index.js +12 -7
- package/build/types/activity.d.ts +3 -3
- package/build/types/exporter.d.ts +2 -2
- package/build/types/exporter.js +0 -6
- package/build/types/hook.d.ts +1 -1
- package/build/types/hotmesh.js +0 -1
- package/build/types/index.d.ts +12 -12
- package/build/types/job.d.ts +1 -1
- package/build/types/logger.js +0 -1
- package/build/types/quorum.d.ts +1 -1
- package/build/types/stats.d.ts +1 -1
- package/build/types/stream.d.ts +1 -2
- package/build/types/telemetry.d.ts +1 -1
- package/build/types/transition.d.ts +1 -1
- package/package.json +41 -38
- package/types/activity.ts +56 -39
- package/types/async.ts +2 -3
- package/types/collator.ts +5 -5
- package/types/durable.ts +161 -161
- package/types/error.ts +37 -37
- package/types/exporter.ts +14 -9
- package/types/hook.ts +11 -4
- package/types/hotmesh.ts +26 -25
- package/types/index.ts +53 -53
- package/types/job.ts +33 -33
- package/types/logger.ts +1 -1
- package/types/map.ts +1 -1
- package/types/pipe.ts +10 -8
- package/types/quorum.ts +20 -13
- package/types/redis.ts +70 -15
- package/types/serializer.ts +8 -6
- package/types/stats.ts +22 -6
- package/types/stream.ts +9 -9
- package/types/task.ts +7 -1
- package/types/telemetry.ts +2 -1
- package/types/transition.ts +8 -8
|
@@ -18,7 +18,10 @@ class Pipe {
|
|
|
18
18
|
return !Array.isArray(currentRow) && '@reduce' in currentRow;
|
|
19
19
|
}
|
|
20
20
|
static isPipeObject(obj) {
|
|
21
|
-
return typeof obj === 'object' &&
|
|
21
|
+
return (typeof obj === 'object' &&
|
|
22
|
+
obj !== null &&
|
|
23
|
+
!Array.isArray(obj) &&
|
|
24
|
+
'@pipe' in obj);
|
|
22
25
|
}
|
|
23
26
|
static resolve(unresolved, context) {
|
|
24
27
|
let pipe;
|
|
@@ -36,7 +39,9 @@ class Pipe {
|
|
|
36
39
|
*/
|
|
37
40
|
process(resolved = null) {
|
|
38
41
|
let index = 0;
|
|
39
|
-
if (!(resolved ||
|
|
42
|
+
if (!(resolved ||
|
|
43
|
+
this.isPipeType(this.rules[0]) ||
|
|
44
|
+
this.isreduceType(this.rules[0]))) {
|
|
40
45
|
resolved = this.processCells(this.rules[0]); // Add type assertion
|
|
41
46
|
index = 1;
|
|
42
47
|
}
|
|
@@ -58,7 +63,7 @@ class Pipe {
|
|
|
58
63
|
return new RegExp(value);
|
|
59
64
|
}
|
|
60
65
|
if (Array.isArray(value)) {
|
|
61
|
-
return value.map(item => this.cloneUnknown(item));
|
|
66
|
+
return value.map((item) => this.cloneUnknown(item));
|
|
62
67
|
}
|
|
63
68
|
const clonedObj = {};
|
|
64
69
|
for (const key in value) {
|
|
@@ -83,15 +88,27 @@ class Pipe {
|
|
|
83
88
|
let resolved = this.cloneUnknown(input[1] ?? null);
|
|
84
89
|
if (Array.isArray(input[0])) {
|
|
85
90
|
for (let index = 0; index < input[0].length; index++) {
|
|
86
|
-
this.context = {
|
|
91
|
+
this.context = {
|
|
92
|
+
$input: input[0],
|
|
93
|
+
$output: resolved,
|
|
94
|
+
$item: input[0][index],
|
|
95
|
+
$key: index.toString(),
|
|
96
|
+
$index: index,
|
|
97
|
+
};
|
|
87
98
|
resolved = this.process([resolved]);
|
|
88
99
|
}
|
|
89
100
|
}
|
|
90
101
|
else {
|
|
91
102
|
let index = -1;
|
|
92
|
-
for (
|
|
103
|
+
for (const $key in input[0]) {
|
|
93
104
|
index++;
|
|
94
|
-
this.context = {
|
|
105
|
+
this.context = {
|
|
106
|
+
$input: input[0],
|
|
107
|
+
$output: resolved,
|
|
108
|
+
$item: input[0][$key],
|
|
109
|
+
$key,
|
|
110
|
+
$index: index,
|
|
111
|
+
};
|
|
95
112
|
resolved = this.process([resolved]);
|
|
96
113
|
}
|
|
97
114
|
}
|
|
@@ -158,15 +175,23 @@ class Pipe {
|
|
|
158
175
|
return resolved;
|
|
159
176
|
}
|
|
160
177
|
isFunction(currentCell) {
|
|
161
|
-
return typeof currentCell === 'string' &&
|
|
178
|
+
return (typeof currentCell === 'string' &&
|
|
179
|
+
currentCell.startsWith('{@') &&
|
|
180
|
+
currentCell.endsWith('}'));
|
|
162
181
|
}
|
|
163
182
|
isContextVariable(currentCell) {
|
|
164
183
|
if (typeof currentCell === 'string' && currentCell.endsWith('}')) {
|
|
165
|
-
return currentCell.startsWith('{$item') ||
|
|
184
|
+
return (currentCell.startsWith('{$item') ||
|
|
185
|
+
currentCell.startsWith('{$key') ||
|
|
186
|
+
currentCell.startsWith('{$index') ||
|
|
187
|
+
currentCell.startsWith('{$input') ||
|
|
188
|
+
currentCell.startsWith('{$output'));
|
|
166
189
|
}
|
|
167
190
|
}
|
|
168
191
|
isMappable(currentCell) {
|
|
169
|
-
return typeof currentCell === 'string' &&
|
|
192
|
+
return (typeof currentCell === 'string' &&
|
|
193
|
+
currentCell.startsWith('{') &&
|
|
194
|
+
currentCell.endsWith('}'));
|
|
170
195
|
}
|
|
171
196
|
resolveCellValue(currentCell) {
|
|
172
197
|
if (this.isFunction(currentCell)) {
|
|
@@ -187,7 +212,9 @@ class Pipe {
|
|
|
187
212
|
const pathParts = path.split('.');
|
|
188
213
|
let current = obj;
|
|
189
214
|
for (const part of pathParts) {
|
|
190
|
-
if (current === null ||
|
|
215
|
+
if (current === null ||
|
|
216
|
+
typeof current !== 'object' ||
|
|
217
|
+
!current.hasOwnProperty(part)) {
|
|
191
218
|
return undefined;
|
|
192
219
|
}
|
|
193
220
|
current = current[part];
|
|
@@ -95,14 +95,16 @@ class QuorumService {
|
|
|
95
95
|
}
|
|
96
96
|
//if there are any callbacks, call them
|
|
97
97
|
if (self.callbacks.length > 0) {
|
|
98
|
-
self.callbacks.forEach(cb => cb(topic, message));
|
|
98
|
+
self.callbacks.forEach((cb) => cb(topic, message));
|
|
99
99
|
}
|
|
100
100
|
};
|
|
101
101
|
}
|
|
102
102
|
async sayPong(appId, guid, originator, details = false) {
|
|
103
103
|
let profile;
|
|
104
104
|
if (details) {
|
|
105
|
-
const stream = this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, {
|
|
105
|
+
const stream = this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, {
|
|
106
|
+
appId: this.appId,
|
|
107
|
+
});
|
|
106
108
|
profile = {
|
|
107
109
|
engine_id: this.guid,
|
|
108
110
|
namespace: this.namespace,
|
|
@@ -119,7 +121,8 @@ class QuorumService {
|
|
|
119
121
|
}
|
|
120
122
|
this.store.publish(hotmesh_1.KeyType.QUORUM, {
|
|
121
123
|
type: 'pong',
|
|
122
|
-
guid,
|
|
124
|
+
guid,
|
|
125
|
+
originator,
|
|
123
126
|
profile,
|
|
124
127
|
}, appId);
|
|
125
128
|
}
|
|
@@ -141,10 +144,10 @@ class QuorumService {
|
|
|
141
144
|
*/
|
|
142
145
|
async doRollCall(message) {
|
|
143
146
|
let iteration = 0;
|
|
144
|
-
|
|
147
|
+
const max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
|
|
145
148
|
if (this.rollCallInterval)
|
|
146
149
|
clearTimeout(this.rollCallInterval);
|
|
147
|
-
const base =
|
|
150
|
+
const base = message.interval / 2;
|
|
148
151
|
const amount = base + Math.ceil(Math.random() * base);
|
|
149
152
|
do {
|
|
150
153
|
await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
|
|
@@ -178,7 +181,7 @@ class QuorumService {
|
|
|
178
181
|
//unsubscribe user from quorum messages
|
|
179
182
|
async unsub(callback) {
|
|
180
183
|
//the quorum is always subscribed to the `quorum` topic; just unregister the fn
|
|
181
|
-
this.callbacks = this.callbacks.filter(cb => cb !== callback);
|
|
184
|
+
this.callbacks = this.callbacks.filter((cb) => cb !== callback);
|
|
182
185
|
}
|
|
183
186
|
// ************* COMPILER METHODS *************
|
|
184
187
|
async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
|
|
@@ -191,13 +194,13 @@ class QuorumService {
|
|
|
191
194
|
this.store.xlen(profile.stream, multi);
|
|
192
195
|
}
|
|
193
196
|
});
|
|
194
|
-
const stream_depths = await multi.exec();
|
|
197
|
+
const stream_depths = (await multi.exec());
|
|
195
198
|
this.profiles.forEach(async (profile) => {
|
|
196
199
|
const index = targetStreams.indexOf(profile.stream);
|
|
197
200
|
if (index != -1) {
|
|
198
|
-
profile.stream_depth = Array.isArray(stream_depths[index])
|
|
199
|
-
stream_depths[index][1]
|
|
200
|
-
stream_depths[index];
|
|
201
|
+
profile.stream_depth = Array.isArray(stream_depths[index])
|
|
202
|
+
? stream_depths[index][1]
|
|
203
|
+
: stream_depths[index];
|
|
201
204
|
}
|
|
202
205
|
});
|
|
203
206
|
return this.profiles;
|
|
@@ -223,7 +226,7 @@ class QuorumService {
|
|
|
223
226
|
if (q1 && q1 === q2 && q2 === q3) {
|
|
224
227
|
this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
|
|
225
228
|
this.store.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
|
|
226
|
-
await new Promise(resolve => setTimeout(resolve, delay));
|
|
229
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
227
230
|
await this.store.releaseScoutRole('activate');
|
|
228
231
|
//confirm we received the activation message
|
|
229
232
|
if (this.engine.untilVersion === version) {
|
|
@@ -21,7 +21,8 @@ class ReporterService {
|
|
|
21
21
|
}
|
|
22
22
|
validateOptions(options) {
|
|
23
23
|
const { start, end, range, granularity } = options;
|
|
24
|
-
if (granularity !== 'infinity' &&
|
|
24
|
+
if (granularity !== 'infinity' &&
|
|
25
|
+
(start && end && range || !start && !end && !range)) {
|
|
25
26
|
throw new Error('Invalid combination of start, end, and range values. Provide either start+end, end+range, or start+range.');
|
|
26
27
|
}
|
|
27
28
|
}
|
|
@@ -61,7 +62,8 @@ class ReporterService {
|
|
|
61
62
|
startTime = new Date(endTime.getTime() - rangeMinutes * 60 * 1000);
|
|
62
63
|
}
|
|
63
64
|
// Round the start time to the nearest granularity unit
|
|
64
|
-
startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) *
|
|
65
|
+
startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) *
|
|
66
|
+
granularityMinutes);
|
|
65
67
|
const dateTimeSets = [];
|
|
66
68
|
for (let time = startTime; time <= endTime; time.setUTCMinutes(time.getUTCMinutes() + granularityMinutes)) {
|
|
67
69
|
const formattedTime = [
|
|
@@ -116,7 +118,7 @@ class ReporterService {
|
|
|
116
118
|
}
|
|
117
119
|
buildStatsResponse(rawData, redisKeys, aggregatedData, count, options) {
|
|
118
120
|
const measures = [];
|
|
119
|
-
const measureKeys = Object.keys(aggregatedData).filter((key) => key !==
|
|
121
|
+
const measureKeys = Object.keys(aggregatedData).filter((key) => key !== 'count');
|
|
120
122
|
let segments = undefined;
|
|
121
123
|
if (options.sparse !== true) {
|
|
122
124
|
segments = this.handleSegments(rawData, redisKeys);
|
|
@@ -124,7 +126,7 @@ class ReporterService {
|
|
|
124
126
|
measureKeys.forEach((key) => {
|
|
125
127
|
const measure = {
|
|
126
128
|
target: key,
|
|
127
|
-
type:
|
|
129
|
+
type: 'count',
|
|
128
130
|
value: aggregatedData[key],
|
|
129
131
|
};
|
|
130
132
|
measures.push(measure);
|
|
@@ -146,12 +148,13 @@ class ReporterService {
|
|
|
146
148
|
const segments = [];
|
|
147
149
|
hashKeys.forEach((hashKey, index) => {
|
|
148
150
|
const segmentData = [];
|
|
149
|
-
data[hashKey] &&
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
data[hashKey] &&
|
|
152
|
+
Object.entries(data[hashKey]).forEach(([key, value]) => {
|
|
153
|
+
if (key.startsWith('count:')) {
|
|
154
|
+
const target = key.slice('count:'.length);
|
|
155
|
+
segmentData.push({ target, type: 'count', value });
|
|
156
|
+
}
|
|
157
|
+
});
|
|
155
158
|
const isoTimestamp = this.isoTimestampFromKeyTimestamp(hashKey);
|
|
156
159
|
const count = data[hashKey] ? data[hashKey].count : 0;
|
|
157
160
|
segments.push({ count, time: isoTimestamp, measures: segmentData });
|
|
@@ -236,7 +239,7 @@ class ReporterService {
|
|
|
236
239
|
return segments;
|
|
237
240
|
}
|
|
238
241
|
getUniqueFacets(data) {
|
|
239
|
-
const targets = data.measures.map(measure => measure.target);
|
|
242
|
+
const targets = data.measures.map((measure) => measure.target);
|
|
240
243
|
return Array.from(new Set(targets));
|
|
241
244
|
}
|
|
242
245
|
getTargetForKey(key) {
|
|
@@ -279,7 +282,7 @@ class ReporterService {
|
|
|
279
282
|
const stats = {
|
|
280
283
|
general: [],
|
|
281
284
|
index: [],
|
|
282
|
-
median: []
|
|
285
|
+
median: [],
|
|
283
286
|
};
|
|
284
287
|
stats.general.push({ metric: 'count', target: 'count', value: 1 });
|
|
285
288
|
for (const measure of statsConfig.measures) {
|
|
@@ -43,12 +43,12 @@ declare class Router {
|
|
|
43
43
|
consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
|
|
44
44
|
isStreamMessage(result: any): boolean;
|
|
45
45
|
consumeOne(stream: string, group: string, id: string, message: string[], callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
|
|
46
|
-
execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<
|
|
46
|
+
execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<StreamData>;
|
|
47
47
|
ackAndDelete(stream: string, group: string, id: string): Promise<void>;
|
|
48
48
|
publishResponse(input: StreamData, output: StreamDataResponse | void): Promise<string>;
|
|
49
49
|
shouldRetry(input: StreamData, output: StreamDataResponse): [boolean, number];
|
|
50
50
|
structureUnhandledError(input: StreamData, err: Error): StreamDataResponse;
|
|
51
|
-
structureUnacknowledgedError(input: StreamData):
|
|
51
|
+
structureUnacknowledgedError(input: StreamData): StreamData;
|
|
52
52
|
structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse;
|
|
53
53
|
static stopConsuming(): Promise<void>;
|
|
54
54
|
stopConsuming(): Promise<void>;
|
|
@@ -44,7 +44,10 @@ class Router {
|
|
|
44
44
|
async publishMessage(topic, streamData, multi) {
|
|
45
45
|
const code = streamData?.code || '200';
|
|
46
46
|
this.counts[code] = (this.counts[code] || 0) + 1;
|
|
47
|
-
const stream = this.store.mintKey(key_1.KeyType.STREAMS, {
|
|
47
|
+
const stream = this.store.mintKey(key_1.KeyType.STREAMS, {
|
|
48
|
+
appId: this.store.appId,
|
|
49
|
+
topic,
|
|
50
|
+
});
|
|
48
51
|
return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
|
|
49
52
|
}
|
|
50
53
|
/**
|
|
@@ -62,7 +65,7 @@ class Router {
|
|
|
62
65
|
if (this.isSleeping)
|
|
63
66
|
return;
|
|
64
67
|
this.isSleeping = true;
|
|
65
|
-
|
|
68
|
+
const startTime = Date.now(); //anchor the origin
|
|
66
69
|
await new Promise(async (outerResolve) => {
|
|
67
70
|
this.sleepPromiseResolve = outerResolve;
|
|
68
71
|
let elapsedTime = Date.now() - startTime;
|
|
@@ -86,12 +89,16 @@ class Router {
|
|
|
86
89
|
async function consume() {
|
|
87
90
|
await this.customSleep();
|
|
88
91
|
if (!this.shouldConsume) {
|
|
89
|
-
this.logger.info(`stream-consumer-stopping`, {
|
|
92
|
+
this.logger.info(`stream-consumer-stopping`, {
|
|
93
|
+
group,
|
|
94
|
+
consumer,
|
|
95
|
+
stream,
|
|
96
|
+
});
|
|
90
97
|
return;
|
|
91
98
|
}
|
|
92
99
|
try {
|
|
93
100
|
//randomizer that asymptotes at 150% of `HMSH_BLOCK_TIME_MS`
|
|
94
|
-
const streamDuration = enums_1.HMSH_BLOCK_TIME_MS + Math.round(
|
|
101
|
+
const streamDuration = enums_1.HMSH_BLOCK_TIME_MS + Math.round(enums_1.HMSH_BLOCK_TIME_MS * Math.random());
|
|
95
102
|
const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', streamDuration, 'STREAMS', stream, '>');
|
|
96
103
|
if (this.isStreamMessage(result)) {
|
|
97
104
|
const [[, messages]] = result;
|
|
@@ -112,9 +119,14 @@ class Router {
|
|
|
112
119
|
}
|
|
113
120
|
catch (err) {
|
|
114
121
|
if (this.shouldConsume && process.env.NODE_ENV !== 'test') {
|
|
115
|
-
this.logger.error(`stream-consume-message-error`, {
|
|
122
|
+
this.logger.error(`stream-consume-message-error`, {
|
|
123
|
+
err,
|
|
124
|
+
stream,
|
|
125
|
+
group,
|
|
126
|
+
consumer,
|
|
127
|
+
});
|
|
116
128
|
this.errorCount++;
|
|
117
|
-
const timeout = Math.min(enums_1.HMSH_GRADUATED_INTERVAL_MS *
|
|
129
|
+
const timeout = Math.min(enums_1.HMSH_GRADUATED_INTERVAL_MS * 2 ** this.errorCount, enums_1.HMSH_MAX_TIMEOUT_MS);
|
|
118
130
|
setTimeout(consume.bind(this), timeout);
|
|
119
131
|
}
|
|
120
132
|
}
|
|
@@ -154,7 +166,15 @@ class Router {
|
|
|
154
166
|
output = await callback(input);
|
|
155
167
|
}
|
|
156
168
|
catch (error) {
|
|
157
|
-
this.logger.error(`stream-call-function-error`, {
|
|
169
|
+
this.logger.error(`stream-call-function-error`, {
|
|
170
|
+
...error,
|
|
171
|
+
input: input,
|
|
172
|
+
stack: error.stack,
|
|
173
|
+
message: error.message,
|
|
174
|
+
name: error.name,
|
|
175
|
+
stream,
|
|
176
|
+
id,
|
|
177
|
+
});
|
|
158
178
|
output = this.structureUnhandledError(input, error);
|
|
159
179
|
}
|
|
160
180
|
return output;
|
|
@@ -171,12 +191,12 @@ class Router {
|
|
|
171
191
|
const [shouldRetry, timeout] = this.shouldRetry(input, output);
|
|
172
192
|
if (shouldRetry) {
|
|
173
193
|
await (0, utils_1.sleepFor)(timeout);
|
|
174
|
-
return await this.publishMessage(input.metadata.topic, {
|
|
194
|
+
return (await this.publishMessage(input.metadata.topic, {
|
|
175
195
|
data: input.data,
|
|
176
196
|
//note: retain guid (this is a retry attempt)
|
|
177
197
|
metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
|
|
178
198
|
policies: input.policies,
|
|
179
|
-
});
|
|
199
|
+
}));
|
|
180
200
|
}
|
|
181
201
|
else {
|
|
182
202
|
output = this.structureError(input, output);
|
|
@@ -189,7 +209,7 @@ class Router {
|
|
|
189
209
|
output.metadata.guid = (0, utils_1.guid)();
|
|
190
210
|
}
|
|
191
211
|
output.type = stream_1.StreamDataType.RESPONSE;
|
|
192
|
-
return await this.publishMessage(null, output);
|
|
212
|
+
return (await this.publishMessage(null, output));
|
|
193
213
|
}
|
|
194
214
|
}
|
|
195
215
|
shouldRetry(input, output) {
|
|
@@ -208,7 +228,7 @@ class Router {
|
|
|
208
228
|
return [false, 0];
|
|
209
229
|
}
|
|
210
230
|
structureUnhandledError(input, err) {
|
|
211
|
-
|
|
231
|
+
const error = {};
|
|
212
232
|
if (typeof err.message === 'string') {
|
|
213
233
|
error.message = err.message;
|
|
214
234
|
}
|
|
@@ -225,7 +245,7 @@ class Router {
|
|
|
225
245
|
status: 'error',
|
|
226
246
|
code: enums_1.HMSH_CODE_UNKNOWN,
|
|
227
247
|
metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
|
|
228
|
-
data: error
|
|
248
|
+
data: error,
|
|
229
249
|
};
|
|
230
250
|
}
|
|
231
251
|
structureUnacknowledgedError(input) {
|
|
@@ -243,10 +263,16 @@ class Router {
|
|
|
243
263
|
return output;
|
|
244
264
|
}
|
|
245
265
|
structureError(input, output) {
|
|
246
|
-
const message = output.data?.message
|
|
266
|
+
const message = output.data?.message
|
|
267
|
+
? output.data?.message.toString()
|
|
268
|
+
: enums_1.HMSH_STATUS_UNKNOWN;
|
|
247
269
|
const statusCode = output.code || output.data?.code;
|
|
248
|
-
const code = isNaN(statusCode)
|
|
249
|
-
|
|
270
|
+
const code = isNaN(statusCode)
|
|
271
|
+
? enums_1.HMSH_CODE_UNKNOWN
|
|
272
|
+
: parseInt(statusCode.toString());
|
|
273
|
+
const stack = output.data?.stack
|
|
274
|
+
? output.data?.stack.toString()
|
|
275
|
+
: undefined;
|
|
250
276
|
const data = { message, code, stack };
|
|
251
277
|
if (typeof output.data?.error === 'object') {
|
|
252
278
|
data.error = { ...output.data.error };
|
|
@@ -256,7 +282,7 @@ class Router {
|
|
|
256
282
|
code,
|
|
257
283
|
stack,
|
|
258
284
|
metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
|
|
259
|
-
data
|
|
285
|
+
data,
|
|
260
286
|
};
|
|
261
287
|
}
|
|
262
288
|
static async stopConsuming() {
|
|
@@ -321,15 +347,27 @@ class Router {
|
|
|
321
347
|
// ii) corrupt hardware/network/transport/etc
|
|
322
348
|
// 3b) system error: Redis unable to accept `xadd` request
|
|
323
349
|
// 4c) system error: Redis unable to accept `xdel`/`xack` request
|
|
324
|
-
this.logger.error('stream-message-max-delivery-count-exceeded', {
|
|
350
|
+
this.logger.error('stream-message-max-delivery-count-exceeded', {
|
|
351
|
+
id,
|
|
352
|
+
stream,
|
|
353
|
+
group,
|
|
354
|
+
consumer,
|
|
355
|
+
code: enums_1.HMSH_CODE_UNACKED,
|
|
356
|
+
count,
|
|
357
|
+
});
|
|
325
358
|
const streamData = reclaimedMessage[0]?.[1]?.[1];
|
|
326
359
|
//fatal risk point 1 of 3): json is corrupt
|
|
327
360
|
const [err, input] = this.parseStreamData(streamData);
|
|
328
361
|
if (err) {
|
|
329
|
-
return this.logger.error('expire-unacknowledged-parse-fatal-error', {
|
|
362
|
+
return this.logger.error('expire-unacknowledged-parse-fatal-error', {
|
|
363
|
+
id,
|
|
364
|
+
err,
|
|
365
|
+
});
|
|
330
366
|
}
|
|
331
367
|
else if (!input || !input.metadata) {
|
|
332
|
-
return this.logger.error('expire-unacknowledged-parse-fatal-error', {
|
|
368
|
+
return this.logger.error('expire-unacknowledged-parse-fatal-error', {
|
|
369
|
+
id,
|
|
370
|
+
});
|
|
333
371
|
}
|
|
334
372
|
let telemetry;
|
|
335
373
|
let messageId;
|
|
@@ -346,12 +384,24 @@ class Router {
|
|
|
346
384
|
}
|
|
347
385
|
catch (err) {
|
|
348
386
|
if (messageId) {
|
|
349
|
-
this.logger.error('expire-unacknowledged-pub-fatal-error', {
|
|
350
|
-
|
|
387
|
+
this.logger.error('expire-unacknowledged-pub-fatal-error', {
|
|
388
|
+
id,
|
|
389
|
+
err,
|
|
390
|
+
...input.metadata,
|
|
391
|
+
});
|
|
392
|
+
telemetry.setStreamAttributes({
|
|
393
|
+
'app.system.fatal': 'expire-unacknowledged-pub-fatal-error',
|
|
394
|
+
});
|
|
351
395
|
}
|
|
352
396
|
else {
|
|
353
|
-
this.logger.error('expire-unacknowledged-ack-fatal-error', {
|
|
354
|
-
|
|
397
|
+
this.logger.error('expire-unacknowledged-ack-fatal-error', {
|
|
398
|
+
id,
|
|
399
|
+
err,
|
|
400
|
+
...input.metadata,
|
|
401
|
+
});
|
|
402
|
+
telemetry.setStreamAttributes({
|
|
403
|
+
'app.system.fatal': 'expire-unacknowledged-ack-fatal-error',
|
|
404
|
+
});
|
|
355
405
|
}
|
|
356
406
|
}
|
|
357
407
|
finally {
|
|
@@ -6,17 +6,37 @@ const dateReg = /^"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?"$/;
|
|
|
6
6
|
exports.MDATA_SYMBOLS = {
|
|
7
7
|
SLOTS: 26,
|
|
8
8
|
ACTIVITY: {
|
|
9
|
-
KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err', 'l1s', 'l2s']
|
|
9
|
+
KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err', 'l1s', 'l2s'],
|
|
10
10
|
},
|
|
11
11
|
ACTIVITY_UPDATE: {
|
|
12
|
-
KEYS: ['au', 'err', 'l2s']
|
|
12
|
+
KEYS: ['au', 'err', 'l2s'],
|
|
13
13
|
},
|
|
14
14
|
JOB: {
|
|
15
|
-
KEYS: [
|
|
15
|
+
KEYS: [
|
|
16
|
+
'ngn',
|
|
17
|
+
'tpc',
|
|
18
|
+
'pj',
|
|
19
|
+
'pg',
|
|
20
|
+
'pd',
|
|
21
|
+
'px',
|
|
22
|
+
'pa',
|
|
23
|
+
'key',
|
|
24
|
+
'app',
|
|
25
|
+
'vrs',
|
|
26
|
+
'jid',
|
|
27
|
+
'gid',
|
|
28
|
+
'aid',
|
|
29
|
+
'ts',
|
|
30
|
+
'jc',
|
|
31
|
+
'ju',
|
|
32
|
+
'js',
|
|
33
|
+
'err',
|
|
34
|
+
'trc',
|
|
35
|
+
],
|
|
16
36
|
},
|
|
17
37
|
JOB_UPDATE: {
|
|
18
|
-
KEYS: ['ju', 'err']
|
|
19
|
-
}
|
|
38
|
+
KEYS: ['ju', 'err'],
|
|
39
|
+
},
|
|
20
40
|
};
|
|
21
41
|
class SerializerService {
|
|
22
42
|
constructor() {
|
|
@@ -72,7 +92,7 @@ class SerializerService {
|
|
|
72
92
|
let map = this.symReverseKeys.get(id);
|
|
73
93
|
if (!map) {
|
|
74
94
|
map = new Map();
|
|
75
|
-
for (
|
|
95
|
+
for (const [key, val] of keyMap.entries()) {
|
|
76
96
|
map.set(val, key);
|
|
77
97
|
}
|
|
78
98
|
this.symReverseKeys.set(id, map);
|
|
@@ -81,16 +101,16 @@ class SerializerService {
|
|
|
81
101
|
}
|
|
82
102
|
getReverseValueMap(valueMap) {
|
|
83
103
|
const map = new Map();
|
|
84
|
-
for (
|
|
104
|
+
for (const [key, val] of valueMap.entries()) {
|
|
85
105
|
map.set(val, key);
|
|
86
106
|
}
|
|
87
107
|
return map;
|
|
88
108
|
}
|
|
89
109
|
static filterSymVals(startIndex, maxIndex, existingSymbolValues, proposedValues) {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
for (
|
|
110
|
+
const newSymbolValues = {};
|
|
111
|
+
const currentSymbolValues = { ...existingSymbolValues };
|
|
112
|
+
const currentValuesSet = new Set(Object.values(currentSymbolValues));
|
|
113
|
+
for (const value of proposedValues) {
|
|
94
114
|
if (!currentValuesSet.has(value)) {
|
|
95
115
|
if (startIndex > maxIndex) {
|
|
96
116
|
return newSymbolValues;
|
|
@@ -107,13 +127,15 @@ class SerializerService {
|
|
|
107
127
|
if (this.symKeys.size === 0) {
|
|
108
128
|
return document;
|
|
109
129
|
}
|
|
110
|
-
|
|
111
|
-
|
|
130
|
+
const source = { ...document };
|
|
131
|
+
const result = {};
|
|
112
132
|
const compressWithMap = (abbreviationMap, id) => {
|
|
113
|
-
for (
|
|
114
|
-
if (key.startsWith(`${id}/`) ||
|
|
133
|
+
for (const key in source) {
|
|
134
|
+
if (key.startsWith(`${id}/`) ||
|
|
135
|
+
id.startsWith('$') &&
|
|
136
|
+
['data', 'metadata'].includes(key.split('/')[0])) {
|
|
115
137
|
const dimensionalIndex = this.resolveDimensionalIndex(key);
|
|
116
|
-
|
|
138
|
+
const shortKey = abbreviationMap.get(key) || key;
|
|
117
139
|
const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
|
|
118
140
|
result[shortDimensionalKey] = source[key];
|
|
119
141
|
}
|
|
@@ -123,7 +145,7 @@ class SerializerService {
|
|
|
123
145
|
}
|
|
124
146
|
}
|
|
125
147
|
};
|
|
126
|
-
for (
|
|
148
|
+
for (const id of ids) {
|
|
127
149
|
const abbreviationMap = this.symKeys.get(id);
|
|
128
150
|
if (abbreviationMap) {
|
|
129
151
|
compressWithMap(abbreviationMap, id);
|
|
@@ -138,20 +160,20 @@ class SerializerService {
|
|
|
138
160
|
if (this.symKeys.size === 0) {
|
|
139
161
|
return document;
|
|
140
162
|
}
|
|
141
|
-
|
|
163
|
+
const result = { ...document };
|
|
142
164
|
const inflateWithMap = (abbreviationMap, id) => {
|
|
143
165
|
const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
|
|
144
|
-
for (
|
|
166
|
+
for (const key in result) {
|
|
145
167
|
//strip dimensional index from key
|
|
146
168
|
const shortKey = key.split(',')[0];
|
|
147
|
-
|
|
169
|
+
const longKey = reversedAbbreviationMap.get(shortKey);
|
|
148
170
|
if (longKey) {
|
|
149
171
|
result[longKey] = result[key];
|
|
150
172
|
delete result[key];
|
|
151
173
|
}
|
|
152
174
|
}
|
|
153
175
|
};
|
|
154
|
-
for (
|
|
176
|
+
for (const id of ids) {
|
|
155
177
|
const abbreviationMap = this.symKeys.get(id);
|
|
156
178
|
if (abbreviationMap) {
|
|
157
179
|
inflateWithMap(abbreviationMap, id);
|
|
@@ -161,9 +183,9 @@ class SerializerService {
|
|
|
161
183
|
}
|
|
162
184
|
//stringify: convert a multi-dimensional document to a 2-d hash
|
|
163
185
|
stringify(document) {
|
|
164
|
-
|
|
165
|
-
for (
|
|
166
|
-
|
|
186
|
+
const result = {};
|
|
187
|
+
for (const key in document) {
|
|
188
|
+
const value = SerializerService.toString(document[key]);
|
|
167
189
|
if (value) {
|
|
168
190
|
// if (/^:*[a-zA-Z]{2}$/.test(value)) {
|
|
169
191
|
// value = ':' + value;
|
|
@@ -177,8 +199,8 @@ class SerializerService {
|
|
|
177
199
|
}
|
|
178
200
|
//parse: convert a 2-d hash to a multi-dimensional document
|
|
179
201
|
parse(document) {
|
|
180
|
-
|
|
181
|
-
for (
|
|
202
|
+
const result = {};
|
|
203
|
+
for (const [key, value] of Object.entries(document)) {
|
|
182
204
|
if (value === undefined || value === null)
|
|
183
205
|
continue;
|
|
184
206
|
// if (/^:+[a-zA-Z]{2}$/.test(value)) {
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
* (a new distributed executable) is deployed to the quorum
|
|
6
6
|
* and the cache is invalidated/cleared of the prior version.
|
|
7
7
|
*/
|
|
8
|
-
import { ActivityType } from
|
|
9
|
-
import { HookRule } from
|
|
10
|
-
import { HotMeshApp, HotMeshSettings } from
|
|
11
|
-
import { Symbols } from
|
|
12
|
-
import { Transitions } from
|
|
8
|
+
import { ActivityType } from '../../types/activity';
|
|
9
|
+
import { HookRule } from '../../types/hook';
|
|
10
|
+
import { HotMeshApp, HotMeshSettings } from '../../types/hotmesh';
|
|
11
|
+
import { Symbols } from '../../types/serializer';
|
|
12
|
+
import { Transitions } from '../../types/transition';
|
|
13
13
|
declare class Cache {
|
|
14
14
|
settings: HotMeshSettings;
|
|
15
15
|
appId: string;
|
|
@@ -110,10 +110,10 @@ class Cache {
|
|
|
110
110
|
this.hookRules[`${appId}`] = hookRules;
|
|
111
111
|
}
|
|
112
112
|
getSignals(appId, version) {
|
|
113
|
-
throw new Error(
|
|
113
|
+
throw new Error('SIGNAL (getHooks) is not supported');
|
|
114
114
|
}
|
|
115
115
|
setSignals(appId, version) {
|
|
116
|
-
throw new Error(
|
|
116
|
+
throw new Error('SIGNAL (setHook) is not supported');
|
|
117
117
|
}
|
|
118
118
|
getActiveTaskQueue(appId) {
|
|
119
119
|
return this.workItems[appId];
|