@hotmeshio/hotmesh 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/ISSUE_TEMPLATE/bug_report.md +38 -0
- package/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
- package/README.md +17 -14
- package/build/modules/enums.d.ts +1 -0
- package/build/modules/enums.js +2 -1
- package/build/modules/key.d.ts +1 -2
- package/build/modules/key.js +3 -4
- package/build/modules/utils.js +10 -11
- package/build/package.json +3 -2
- package/build/services/activities/activity.js +1 -1
- package/build/services/activities/await.js +1 -1
- package/build/services/activities/cycle.js +1 -1
- package/build/services/activities/hook.js +1 -1
- package/build/services/activities/interrupt.js +1 -1
- package/build/services/activities/signal.js +1 -1
- package/build/services/activities/trigger.d.ts +2 -1
- package/build/services/activities/trigger.js +21 -9
- package/build/services/activities/worker.js +1 -1
- package/build/services/compiler/deployer.js +1 -1
- package/build/services/durable/client.js +5 -1
- package/build/services/durable/worker.js +1 -1
- package/build/services/engine/index.d.ts +1 -1
- package/build/services/engine/index.js +13 -6
- package/build/services/hotmesh/index.js +13 -1
- package/build/services/router/index.d.ts +2 -0
- package/build/services/router/index.js +37 -15
- package/build/services/store/clients/ioredis.js +28 -8
- package/build/services/store/clients/redis.js +27 -10
- package/build/services/store/index.d.ts +6 -2
- package/build/services/store/index.js +37 -5
- package/build/services/stream/clients/ioredis.js +1 -1
- package/build/services/stream/clients/redis.js +23 -23
- package/build/services/worker/index.d.ts +1 -1
- package/build/services/worker/index.js +4 -2
- package/build/types/durable.d.ts +11 -0
- package/build/types/hotmesh.d.ts +1 -1
- package/build/types/hotmesh.js +1 -1
- package/build/types/job.d.ts +11 -0
- package/build/types/quorum.d.ts +2 -4
- package/build/types/redis.d.ts +3 -0
- package/build/types/stream.d.ts +2 -0
- package/package.json +3 -2
- package/types/durable.ts +14 -1
- package/types/hotmesh.ts +1 -1
- package/types/job.ts +12 -0
- package/types/quorum.ts +2 -4
- package/types/redis.ts +3 -0
- package/types/stream.ts +2 -0
|
@@ -22,6 +22,7 @@ class Router {
|
|
|
22
22
|
this.topic = config.topic;
|
|
23
23
|
this.stream = stream;
|
|
24
24
|
this.store = store;
|
|
25
|
+
this.throttle = config.throttle;
|
|
25
26
|
this.reclaimDelay = config.reclaimDelay || enums_1.HMSH_XCLAIM_DELAY_MS;
|
|
26
27
|
this.reclaimCount = config.reclaimCount || enums_1.HMSH_XCLAIM_COUNT;
|
|
27
28
|
this.logger = logger;
|
|
@@ -38,7 +39,7 @@ class Router {
|
|
|
38
39
|
await this.store.xgroup('CREATE', stream, group, '$', 'MKSTREAM');
|
|
39
40
|
}
|
|
40
41
|
catch (err) {
|
|
41
|
-
this.logger.debug('
|
|
42
|
+
this.logger.debug('router-stream-group-exists', { stream, group });
|
|
42
43
|
}
|
|
43
44
|
}
|
|
44
45
|
async publishMessage(topic, streamData, multi) {
|
|
@@ -81,25 +82,31 @@ class Router {
|
|
|
81
82
|
});
|
|
82
83
|
}
|
|
83
84
|
async consumeMessages(stream, group, consumer, callback) {
|
|
84
|
-
this.logger.info(`stream-
|
|
85
|
+
this.logger.info(`router-stream-starting`, { group, consumer, stream });
|
|
85
86
|
Router.instances.add(this);
|
|
86
87
|
this.shouldConsume = true;
|
|
87
88
|
await this.createGroup(stream, group);
|
|
88
89
|
let lastCheckedPendingMessagesAt = Date.now();
|
|
89
90
|
async function consume() {
|
|
90
91
|
await this.customSleep();
|
|
91
|
-
if (
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
});
|
|
92
|
+
if (this.isStopped(group, consumer, stream)) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
else if (this.isPaused()) {
|
|
96
|
+
setImmediate(consume.bind(this));
|
|
97
97
|
return;
|
|
98
98
|
}
|
|
99
99
|
try {
|
|
100
100
|
//randomizer that asymptotes at 150% of `HMSH_BLOCK_TIME_MS`
|
|
101
101
|
const streamDuration = enums_1.HMSH_BLOCK_TIME_MS + Math.round(enums_1.HMSH_BLOCK_TIME_MS * Math.random());
|
|
102
102
|
const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', streamDuration, 'STREAMS', stream, '>');
|
|
103
|
+
if (this.isStopped(group, consumer, stream)) {
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
else if (this.isPaused()) {
|
|
107
|
+
setImmediate(consume.bind(this));
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
103
110
|
if (this.isStreamMessage(result)) {
|
|
104
111
|
const [[, messages]] = result;
|
|
105
112
|
for (const [id, message] of messages) {
|
|
@@ -119,7 +126,7 @@ class Router {
|
|
|
119
126
|
}
|
|
120
127
|
catch (err) {
|
|
121
128
|
if (this.shouldConsume && process.env.NODE_ENV !== 'test') {
|
|
122
|
-
this.logger.error(`stream-
|
|
129
|
+
this.logger.error(`router-stream-error`, {
|
|
123
130
|
err,
|
|
124
131
|
stream,
|
|
125
132
|
group,
|
|
@@ -136,8 +143,21 @@ class Router {
|
|
|
136
143
|
isStreamMessage(result) {
|
|
137
144
|
return Array.isArray(result) && Array.isArray(result[0]);
|
|
138
145
|
}
|
|
146
|
+
isPaused() {
|
|
147
|
+
return this.throttle === enums_1.MAX_DELAY;
|
|
148
|
+
}
|
|
149
|
+
isStopped(group, consumer, stream) {
|
|
150
|
+
if (!this.shouldConsume) {
|
|
151
|
+
this.logger.info(`router-stream-stopped`, {
|
|
152
|
+
group,
|
|
153
|
+
consumer,
|
|
154
|
+
stream,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
return !this.shouldConsume;
|
|
158
|
+
}
|
|
139
159
|
async consumeOne(stream, group, id, message, callback) {
|
|
140
|
-
this.logger.debug(`stream-
|
|
160
|
+
this.logger.debug(`stream-read-one`, { group, stream, id });
|
|
141
161
|
const [err, input] = this.parseStreamData(message[1]);
|
|
142
162
|
let output;
|
|
143
163
|
let telemetry;
|
|
@@ -151,14 +171,14 @@ class Router {
|
|
|
151
171
|
this.errorCount = 0;
|
|
152
172
|
}
|
|
153
173
|
catch (err) {
|
|
154
|
-
this.logger.error(`stream-
|
|
174
|
+
this.logger.error(`stream-read-one-error`, { group, stream, id, err });
|
|
155
175
|
telemetry.setStreamError(err.message);
|
|
156
176
|
}
|
|
157
177
|
const messageId = await this.publishResponse(input, output);
|
|
158
178
|
telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
|
|
159
179
|
await this.ackAndDelete(stream, group, id);
|
|
160
180
|
telemetry.endStreamSpan();
|
|
161
|
-
this.logger.debug(`stream-
|
|
181
|
+
this.logger.debug(`stream-read-one-end`, { group, stream, id });
|
|
162
182
|
}
|
|
163
183
|
async execStreamLeg(input, stream, id, callback) {
|
|
164
184
|
let output;
|
|
@@ -293,7 +313,7 @@ class Router {
|
|
|
293
313
|
}
|
|
294
314
|
async stopConsuming() {
|
|
295
315
|
this.shouldConsume = false;
|
|
296
|
-
this.logger.info(`stream-
|
|
316
|
+
this.logger.info(`router-stream-stopping`, this.topic ? { topic: this.topic } : undefined);
|
|
297
317
|
this.cancelThrottle();
|
|
298
318
|
}
|
|
299
319
|
cancelThrottle() {
|
|
@@ -303,8 +323,10 @@ class Router {
|
|
|
303
323
|
this.resetThrottleState();
|
|
304
324
|
}
|
|
305
325
|
setThrottle(delayInMillis) {
|
|
306
|
-
if (!Number.isInteger(delayInMillis) ||
|
|
307
|
-
|
|
326
|
+
if (!Number.isInteger(delayInMillis) ||
|
|
327
|
+
delayInMillis < 0 ||
|
|
328
|
+
delayInMillis > enums_1.MAX_DELAY) {
|
|
329
|
+
throw new Error(`Throttle must be a non-negative integer and not exceed ${enums_1.MAX_DELAY} ms; send -1 to throttle indefinitely`);
|
|
308
330
|
}
|
|
309
331
|
const wasDecreased = delayInMillis < this.throttle;
|
|
310
332
|
this.throttle = delayInMillis;
|
|
@@ -29,17 +29,17 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
29
29
|
async exec() {
|
|
30
30
|
if (commands.length === 0)
|
|
31
31
|
return [];
|
|
32
|
-
const sameKey = commands.every(cmd => {
|
|
32
|
+
const sameKey = commands.every((cmd) => {
|
|
33
33
|
return cmd.args[0] === commands[0].args[0];
|
|
34
34
|
});
|
|
35
35
|
if (sameKey) {
|
|
36
36
|
const multi = my.redisClient.multi();
|
|
37
|
-
commands.forEach(cmd => multi[cmd.command](...cmd.args));
|
|
37
|
+
commands.forEach((cmd) => multi[cmd.command](...cmd.args));
|
|
38
38
|
const results = await multi.exec();
|
|
39
|
-
return results.map(item => item);
|
|
39
|
+
return results.map((item) => item);
|
|
40
40
|
}
|
|
41
41
|
else {
|
|
42
|
-
return Promise.all(commands.map(cmd => my.redisClient[cmd.command](...cmd.args)));
|
|
42
|
+
return Promise.all(commands.map((cmd) => my.redisClient[cmd.command](...cmd.args)));
|
|
43
43
|
}
|
|
44
44
|
},
|
|
45
45
|
xadd(key, id, fields, message) {
|
|
@@ -55,10 +55,30 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
55
55
|
return addCommand('xlen', [key]);
|
|
56
56
|
},
|
|
57
57
|
xpending(key, group, start, end, count, consumer) {
|
|
58
|
-
return addCommand('xpending', [
|
|
58
|
+
return addCommand('xpending', [
|
|
59
|
+
key,
|
|
60
|
+
group,
|
|
61
|
+
start,
|
|
62
|
+
end,
|
|
63
|
+
count,
|
|
64
|
+
consumer,
|
|
65
|
+
]);
|
|
59
66
|
},
|
|
60
67
|
xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
61
|
-
return addCommand('xclaim', [
|
|
68
|
+
return addCommand('xclaim', [
|
|
69
|
+
key,
|
|
70
|
+
group,
|
|
71
|
+
consumer,
|
|
72
|
+
minIdleTime,
|
|
73
|
+
id,
|
|
74
|
+
...args,
|
|
75
|
+
]);
|
|
76
|
+
},
|
|
77
|
+
del(key) {
|
|
78
|
+
return addCommand('del', [key]);
|
|
79
|
+
},
|
|
80
|
+
expire: function (key, seconds) {
|
|
81
|
+
return addCommand('expire', [key, seconds]);
|
|
62
82
|
},
|
|
63
83
|
hdel(key, itemId) {
|
|
64
84
|
return addCommand('hdel', [key, itemId]);
|
|
@@ -89,7 +109,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
89
109
|
},
|
|
90
110
|
xgroup(command, key, groupName, id, mkStream) {
|
|
91
111
|
return addCommand('xgroup', [command, key, groupName, id, mkStream]);
|
|
92
|
-
}
|
|
112
|
+
},
|
|
93
113
|
};
|
|
94
114
|
return multiInstance;
|
|
95
115
|
}
|
|
@@ -131,7 +151,7 @@ class IORedisStoreService extends index_1.StoreService {
|
|
|
131
151
|
return ((await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK');
|
|
132
152
|
}
|
|
133
153
|
catch (err) {
|
|
134
|
-
this.logger.debug(
|
|
154
|
+
this.logger.debug('stream-mkstream-caught', { key, group: groupName });
|
|
135
155
|
throw err;
|
|
136
156
|
}
|
|
137
157
|
}
|
|
@@ -58,22 +58,22 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
58
58
|
async exec() {
|
|
59
59
|
if (commands.length === 0)
|
|
60
60
|
return [];
|
|
61
|
-
const sameKey = commands.every(cmd => {
|
|
61
|
+
const sameKey = commands.every((cmd) => {
|
|
62
62
|
return cmd.args[0] === commands[0].args[0];
|
|
63
63
|
});
|
|
64
64
|
if (sameKey) {
|
|
65
65
|
const multi = my.redisClient.multi();
|
|
66
|
-
commands.forEach(cmd => {
|
|
66
|
+
commands.forEach((cmd) => {
|
|
67
67
|
if (cmd.command === 'ZADD') {
|
|
68
68
|
return multi.ZADD(cmd.args[0], cmd.args[1], cmd.args[2]);
|
|
69
69
|
}
|
|
70
70
|
return multi[cmd.command](...cmd.args);
|
|
71
71
|
});
|
|
72
72
|
const results = await multi.exec();
|
|
73
|
-
return results.map(item => item);
|
|
73
|
+
return results.map((item) => item);
|
|
74
74
|
}
|
|
75
75
|
else {
|
|
76
|
-
return Promise.all(commands.map(cmd => {
|
|
76
|
+
return Promise.all(commands.map((cmd) => {
|
|
77
77
|
if (cmd.command === 'ZADD') {
|
|
78
78
|
return my.redisClient.ZADD(cmd.args[0], cmd.args[1], cmd.args[2]);
|
|
79
79
|
}
|
|
@@ -94,10 +94,30 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
94
94
|
return addCommand('XLEN', [key]);
|
|
95
95
|
},
|
|
96
96
|
XCLAIM(key, group, consumer, minIdleTime, id, ...args) {
|
|
97
|
-
return addCommand('XCLAIM', [
|
|
97
|
+
return addCommand('XCLAIM', [
|
|
98
|
+
key,
|
|
99
|
+
group,
|
|
100
|
+
consumer,
|
|
101
|
+
minIdleTime,
|
|
102
|
+
id,
|
|
103
|
+
...args,
|
|
104
|
+
]);
|
|
98
105
|
},
|
|
99
106
|
XPENDING(key, group, start, end, count, consumer) {
|
|
100
|
-
return addCommand('XPENDING', [
|
|
107
|
+
return addCommand('XPENDING', [
|
|
108
|
+
key,
|
|
109
|
+
group,
|
|
110
|
+
start,
|
|
111
|
+
end,
|
|
112
|
+
count,
|
|
113
|
+
consumer,
|
|
114
|
+
]);
|
|
115
|
+
},
|
|
116
|
+
DEL: function (key) {
|
|
117
|
+
return addCommand('DEL', [key]);
|
|
118
|
+
},
|
|
119
|
+
EXPIRE: function (key, seconds) {
|
|
120
|
+
return addCommand('EXPIRE', [key, seconds]);
|
|
101
121
|
},
|
|
102
122
|
HDEL(key, itemId) {
|
|
103
123
|
return addCommand('HDEL', [key, itemId]);
|
|
@@ -129,9 +149,6 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
129
149
|
XGROUP(command, key, groupName, id, mkStream) {
|
|
130
150
|
return addCommand('XGROUP', [command, key, groupName, id, mkStream]);
|
|
131
151
|
},
|
|
132
|
-
DEL: function (key) {
|
|
133
|
-
throw new Error('Function not implemented.');
|
|
134
|
-
},
|
|
135
152
|
EXISTS: function (key) {
|
|
136
153
|
throw new Error('Function not implemented.');
|
|
137
154
|
},
|
|
@@ -152,7 +169,7 @@ class RedisStoreService extends index_1.StoreService {
|
|
|
152
169
|
},
|
|
153
170
|
ZSCORE: function (key, value) {
|
|
154
171
|
throw new Error('Function not implemented.');
|
|
155
|
-
}
|
|
172
|
+
},
|
|
156
173
|
};
|
|
157
174
|
return multiInstance;
|
|
158
175
|
}
|
|
@@ -11,6 +11,7 @@ import { Transitions } from '../../types/transition';
|
|
|
11
11
|
import { ReclaimedMessageType } from '../../types/stream';
|
|
12
12
|
import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
|
|
13
13
|
import { WorkListTaskType } from '../../types/task';
|
|
14
|
+
import { ThrottleOptions } from '../../types/quorum';
|
|
14
15
|
import { Cache } from './cache';
|
|
15
16
|
interface AbstractRedisClient {
|
|
16
17
|
exec(): any;
|
|
@@ -101,7 +102,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
101
102
|
* purposeful re-entry.
|
|
102
103
|
*/
|
|
103
104
|
collateSynthetic(jobId: string, guid: string, amount: number, multi?: U): Promise<number>;
|
|
104
|
-
setStateNX(jobId: string, appId: string): Promise<boolean>;
|
|
105
|
+
setStateNX(jobId: string, appId: string, status?: number): Promise<boolean>;
|
|
105
106
|
getSchema(activityId: string, appVersion: AppVID): Promise<ActivityType>;
|
|
106
107
|
getSchemas(appVersion: AppVID): Promise<Record<string, ActivityType>>;
|
|
107
108
|
setSchemas(schemas: Record<string, ActivityType>, appVersion: AppVID): Promise<any>;
|
|
@@ -119,7 +120,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
119
120
|
getActiveTaskQueue(): Promise<string | null>;
|
|
120
121
|
deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
|
|
121
122
|
processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
|
|
122
|
-
expireJob(jobId: string, inSeconds: number): Promise<void>;
|
|
123
|
+
expireJob(jobId: string, inSeconds: number, redisMulti?: U): Promise<void>;
|
|
123
124
|
/**
|
|
124
125
|
* register the descendants of an expired origin flow to be
|
|
125
126
|
* expired at a future date; options indicate whether this
|
|
@@ -162,5 +163,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
162
163
|
scrub(jobId: string): Promise<void>;
|
|
163
164
|
findJobs(queryString?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, string[]]>;
|
|
164
165
|
findJobFields(jobId: string, fieldMatchPattern?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, StringStringType]>;
|
|
166
|
+
setThrottleRate(options: ThrottleOptions): Promise<void>;
|
|
167
|
+
getThrottleRates(): Promise<StringStringType>;
|
|
168
|
+
getThrottleRate(topic: string): Promise<number>;
|
|
165
169
|
}
|
|
166
170
|
export { StoreService };
|
|
@@ -24,11 +24,11 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
exports.StoreService = void 0;
|
|
27
|
+
const errors_1 = require("../../modules/errors");
|
|
27
28
|
const key_1 = require("../../modules/key");
|
|
28
29
|
const serializer_1 = require("../serializer");
|
|
29
30
|
const utils_1 = require("../../modules/utils");
|
|
30
31
|
const enums_1 = require("../../modules/enums");
|
|
31
|
-
const errors_1 = require("../../modules/errors");
|
|
32
32
|
const cache_1 = require("./cache");
|
|
33
33
|
class StoreService {
|
|
34
34
|
constructor(redisClient) {
|
|
@@ -593,9 +593,9 @@ class StoreService {
|
|
|
593
593
|
});
|
|
594
594
|
return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount);
|
|
595
595
|
}
|
|
596
|
-
async setStateNX(jobId, appId) {
|
|
596
|
+
async setStateNX(jobId, appId, status) {
|
|
597
597
|
const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
|
|
598
|
-
const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', '1');
|
|
598
|
+
const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', status?.toString() ?? '1');
|
|
599
599
|
return this.isSuccessful(result);
|
|
600
600
|
}
|
|
601
601
|
async getSchema(activityId, appVersion) {
|
|
@@ -803,13 +803,13 @@ class StoreService {
|
|
|
803
803
|
async processTaskQueue(sourceKey, destinationKey) {
|
|
804
804
|
return await this.redisClient[this.commands.lmove](sourceKey, destinationKey, 'LEFT', 'RIGHT');
|
|
805
805
|
}
|
|
806
|
-
async expireJob(jobId, inSeconds) {
|
|
806
|
+
async expireJob(jobId, inSeconds, redisMulti) {
|
|
807
807
|
if (!isNaN(inSeconds) && inSeconds > 0) {
|
|
808
808
|
const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
|
|
809
809
|
appId: this.appId,
|
|
810
810
|
jobId,
|
|
811
811
|
});
|
|
812
|
-
await this.redisClient[this.commands.expire](jobKey, inSeconds);
|
|
812
|
+
await (redisMulti || this.redisClient)[this.commands.expire](jobKey, inSeconds);
|
|
813
813
|
}
|
|
814
814
|
}
|
|
815
815
|
/**
|
|
@@ -1004,5 +1004,37 @@ class StoreService {
|
|
|
1004
1004
|
} while (cursor !== '0' && len < limit);
|
|
1005
1005
|
return [cursor, matchingFields];
|
|
1006
1006
|
}
|
|
1007
|
+
async setThrottleRate(options) {
|
|
1008
|
+
const key = this.mintKey(key_1.KeyType.THROTTLE_RATE, { appId: this.appId });
|
|
1009
|
+
//engine guids are session specific. no need to persist
|
|
1010
|
+
if (options.guid) {
|
|
1011
|
+
return;
|
|
1012
|
+
}
|
|
1013
|
+
//if a topic, update
|
|
1014
|
+
if (options.topic) {
|
|
1015
|
+
await this.redisClient[this.commands.hset](key, options.topic, options.throttle.toString());
|
|
1016
|
+
}
|
|
1017
|
+
else {
|
|
1018
|
+
//if no topic, update all
|
|
1019
|
+
const multi = this.getMulti();
|
|
1020
|
+
multi[this.commands.del](key);
|
|
1021
|
+
multi[this.commands.hset](key, ':', options.throttle.toString());
|
|
1022
|
+
await multi.exec();
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
async getThrottleRates() {
|
|
1026
|
+
const key = this.mintKey(key_1.KeyType.THROTTLE_RATE, { appId: this.appId });
|
|
1027
|
+
const response = await this.redisClient[this.commands.hgetall](key);
|
|
1028
|
+
return response ?? {};
|
|
1029
|
+
}
|
|
1030
|
+
async getThrottleRate(topic) {
|
|
1031
|
+
const response = await this.getThrottleRates();
|
|
1032
|
+
const rate = topic in response ? Number(response[topic]) : 0;
|
|
1033
|
+
if (isNaN(rate))
|
|
1034
|
+
return 0;
|
|
1035
|
+
if (rate == -1)
|
|
1036
|
+
return enums_1.MAX_DELAY;
|
|
1037
|
+
return Math.max(Math.min(rate, enums_1.MAX_DELAY), 0);
|
|
1038
|
+
}
|
|
1007
1039
|
}
|
|
1008
1040
|
exports.StoreService = StoreService;
|
|
@@ -26,7 +26,7 @@ class IORedisStreamService extends index_1.StreamService {
|
|
|
26
26
|
return ((await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK');
|
|
27
27
|
}
|
|
28
28
|
catch (error) {
|
|
29
|
-
this.logger.
|
|
29
|
+
this.logger.debug('stream-mkstream-caught', { key, group: groupName });
|
|
30
30
|
throw error;
|
|
31
31
|
}
|
|
32
32
|
}
|
|
@@ -32,10 +32,10 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
32
32
|
...args,
|
|
33
33
|
])) === 1);
|
|
34
34
|
}
|
|
35
|
-
catch (
|
|
35
|
+
catch (error) {
|
|
36
36
|
const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
|
|
37
|
-
this.logger.error(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`,
|
|
38
|
-
throw
|
|
37
|
+
this.logger.error(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { ...error });
|
|
38
|
+
throw error;
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
async xadd(key, id, ...args) {
|
|
@@ -48,9 +48,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
48
48
|
[args[0]]: args[1],
|
|
49
49
|
});
|
|
50
50
|
}
|
|
51
|
-
catch (
|
|
52
|
-
this.logger.error(`Error publishing 'xadd'; key: ${key}`,
|
|
53
|
-
throw
|
|
51
|
+
catch (error) {
|
|
52
|
+
this.logger.error(`Error publishing 'xadd'; key: ${key}`, { ...error });
|
|
53
|
+
throw error;
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
async xreadgroup(command, groupName, consumerName, blockOption, blockTime, streamsOption, streamName, id) {
|
|
@@ -67,9 +67,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
67
67
|
id,
|
|
68
68
|
]);
|
|
69
69
|
}
|
|
70
|
-
catch (
|
|
71
|
-
this.logger.error(`Error
|
|
72
|
-
throw
|
|
70
|
+
catch (error) {
|
|
71
|
+
this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
|
|
72
|
+
throw error;
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
async xpending(key, group, start, end, count, consumer) {
|
|
@@ -86,13 +86,13 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
86
86
|
try {
|
|
87
87
|
return await this.redisClient.sendCommand(['XPENDING', ...args]);
|
|
88
88
|
}
|
|
89
|
-
catch (
|
|
90
|
-
this.logger.error('
|
|
89
|
+
catch (error) {
|
|
90
|
+
this.logger.error('error, args', { ...error }, args);
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
|
-
catch (
|
|
94
|
-
this.logger.error(`Error
|
|
95
|
-
throw
|
|
93
|
+
catch (error) {
|
|
94
|
+
this.logger.error(`Error retrieving pending messages for group: ${group} in key: ${key}`, { ...error });
|
|
95
|
+
throw error;
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
async xclaim(key, group, consumer, minIdleTime, id, ...args) {
|
|
@@ -107,9 +107,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
107
107
|
...args,
|
|
108
108
|
]));
|
|
109
109
|
}
|
|
110
|
-
catch (
|
|
111
|
-
this.logger.error(`Error
|
|
112
|
-
throw
|
|
110
|
+
catch (error) {
|
|
111
|
+
this.logger.error(`Error claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
|
|
112
|
+
throw error;
|
|
113
113
|
}
|
|
114
114
|
}
|
|
115
115
|
async xack(key, group, id, multi) {
|
|
@@ -122,9 +122,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
122
122
|
return await this.redisClient.XACK(key, group, id);
|
|
123
123
|
}
|
|
124
124
|
}
|
|
125
|
-
catch (
|
|
126
|
-
this.logger.error(`Error
|
|
127
|
-
throw
|
|
125
|
+
catch (error) {
|
|
126
|
+
this.logger.error(`Error acknowledging messages in group: ${group} for key: ${key}`, { ...error });
|
|
127
|
+
throw error;
|
|
128
128
|
}
|
|
129
129
|
}
|
|
130
130
|
async xdel(key, id, multi) {
|
|
@@ -137,9 +137,9 @@ class RedisStreamService extends index_1.StreamService {
|
|
|
137
137
|
return await this.redisClient.XDEL(key, id);
|
|
138
138
|
}
|
|
139
139
|
}
|
|
140
|
-
catch (
|
|
141
|
-
this.logger.error(`Error
|
|
142
|
-
throw
|
|
140
|
+
catch (error) {
|
|
141
|
+
this.logger.error(`Error deleting messages with ids: ${id} for key: ${key}`, { ...error });
|
|
142
|
+
throw error;
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
async xlen(key, multi) {
|
|
@@ -28,7 +28,7 @@ declare class WorkerService {
|
|
|
28
28
|
initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;
|
|
29
29
|
initSubChannel(service: WorkerService, sub: RedisClient): Promise<void>;
|
|
30
30
|
initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
|
|
31
|
-
initRouter(worker: HotMeshWorker, logger: ILogger): Router
|
|
31
|
+
initRouter(worker: HotMeshWorker, logger: ILogger): Promise<Router>;
|
|
32
32
|
subscriptionHandler(): SubscriptionCallback;
|
|
33
33
|
/**
|
|
34
34
|
* A quorum-wide command to broadcaset system details.
|
|
@@ -37,7 +37,7 @@ class WorkerService {
|
|
|
37
37
|
await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId, service.topic);
|
|
38
38
|
await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId, service.guid);
|
|
39
39
|
await service.initStreamChannel(service, worker.stream);
|
|
40
|
-
service.router = service.initRouter(worker, logger);
|
|
40
|
+
service.router = await service.initRouter(worker, logger);
|
|
41
41
|
const key = service.stream.mintKey(key_1.KeyType.STREAMS, {
|
|
42
42
|
appId: service.appId,
|
|
43
43
|
topic: worker.topic,
|
|
@@ -84,7 +84,8 @@ class WorkerService {
|
|
|
84
84
|
}
|
|
85
85
|
await service.stream.init(service.namespace, service.appId, service.logger);
|
|
86
86
|
}
|
|
87
|
-
initRouter(worker, logger) {
|
|
87
|
+
async initRouter(worker, logger) {
|
|
88
|
+
const throttle = await this.store.getThrottleRate(worker.topic);
|
|
88
89
|
return new router_1.Router({
|
|
89
90
|
namespace: this.namespace,
|
|
90
91
|
appId: this.appId,
|
|
@@ -93,6 +94,7 @@ class WorkerService {
|
|
|
93
94
|
topic: worker.topic,
|
|
94
95
|
reclaimDelay: worker.reclaimDelay,
|
|
95
96
|
reclaimCount: worker.reclaimCount,
|
|
97
|
+
throttle,
|
|
96
98
|
}, this.stream, this.store, logger);
|
|
97
99
|
}
|
|
98
100
|
subscriptionHandler() {
|
package/build/types/durable.d.ts
CHANGED
|
@@ -260,6 +260,17 @@ type WorkflowOptions = {
|
|
|
260
260
|
* default is true; if false, will not await the execution
|
|
261
261
|
*/
|
|
262
262
|
await?: boolean;
|
|
263
|
+
/**
|
|
264
|
+
* If provided, the job will initialize in an expired state, reserving
|
|
265
|
+
* only the job ID (HSETNX) and persisting search and marker (if provided).
|
|
266
|
+
* If a `resume` signal is sent before the specified number of seconds,
|
|
267
|
+
* the job will resume as normal, transition to the adjacent children
|
|
268
|
+
* of the trigger. If the job is not resumed within the number
|
|
269
|
+
* of seconds specified, the job will be scrubbed. No dependencies
|
|
270
|
+
* are added for a job in an expired state; however, dependencies
|
|
271
|
+
* will be added after the job is resumed if relevant.
|
|
272
|
+
*/
|
|
273
|
+
expired?: number;
|
|
263
274
|
};
|
|
264
275
|
/**
|
|
265
276
|
* Options for setting up a hook.
|
package/build/types/hotmesh.d.ts
CHANGED
package/build/types/hotmesh.js
CHANGED
|
@@ -7,7 +7,7 @@ exports.KeyType = void 0;
|
|
|
7
7
|
var KeyType;
|
|
8
8
|
(function (KeyType) {
|
|
9
9
|
KeyType["APP"] = "APP";
|
|
10
|
-
KeyType["
|
|
10
|
+
KeyType["THROTTLE_RATE"] = "THROTTLE_RATE";
|
|
11
11
|
KeyType["HOOKS"] = "HOOKS";
|
|
12
12
|
KeyType["JOB_DEPENDENTS"] = "JOB_DEPENDENTS";
|
|
13
13
|
KeyType["JOB_STATE"] = "JOB_STATE";
|
package/build/types/job.d.ts
CHANGED
|
@@ -77,6 +77,17 @@ type ExtensionType = {
|
|
|
77
77
|
* the initial data set.
|
|
78
78
|
*/
|
|
79
79
|
marker?: StringStringType;
|
|
80
|
+
/**
|
|
81
|
+
* If provided, the job will initialize in an expired state, reserving
|
|
82
|
+
* only the job ID (HSETNX) and persisting search and marker (if provided).
|
|
83
|
+
* If a `resume` signal is sent before the specified number of seconds,
|
|
84
|
+
* the job will resume as normal, transition the the adjacent children
|
|
85
|
+
* of the trigger. If the job is not resumed within the
|
|
86
|
+
* number of seconds specified, the job will be scrubbed. No dependencies
|
|
87
|
+
* are added for a job in an expired state; however, dependencies will be
|
|
88
|
+
* added after the job is resumed if relevant.
|
|
89
|
+
*/
|
|
90
|
+
expired?: number;
|
|
80
91
|
};
|
|
81
92
|
/**
|
|
82
93
|
* job_status semaphore
|
package/build/types/quorum.d.ts
CHANGED
|
@@ -30,11 +30,9 @@ export type ThrottleOptions = {
|
|
|
30
30
|
topic?: string;
|
|
31
31
|
/** entity/noun */
|
|
32
32
|
entity?: string;
|
|
33
|
-
/** in milliseconds;
|
|
33
|
+
/** in milliseconds; set to -1 for indefinite throttle */
|
|
34
34
|
throttle: number;
|
|
35
|
-
/** namespace
|
|
36
|
-
* @default 'durable'
|
|
37
|
-
*/
|
|
35
|
+
/** namespace */
|
|
38
36
|
namespace?: string;
|
|
39
37
|
};
|
|
40
38
|
export interface QuorumProfile {
|
package/build/types/redis.d.ts
CHANGED
|
@@ -68,6 +68,7 @@ interface RedisRedisMultiType {
|
|
|
68
68
|
XLEN(key: string): this;
|
|
69
69
|
DEL(key: string): this;
|
|
70
70
|
EXISTS(key: string): this;
|
|
71
|
+
EXPIRE(key: string, seconds: number): this;
|
|
71
72
|
HDEL(key: string, itemId: string): this;
|
|
72
73
|
HGET(key: string, itemId: string): this;
|
|
73
74
|
HGETALL(key: string): this;
|
|
@@ -215,6 +216,8 @@ interface IORedisMultiType {
|
|
|
215
216
|
xlen(key: string): this;
|
|
216
217
|
xpending(key: string, group: string, start?: string, end?: string, count?: number, consumer?: string): this;
|
|
217
218
|
xclaim(key: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): this;
|
|
219
|
+
del(key: string): this;
|
|
220
|
+
expire(key: string, seconds: number): this;
|
|
218
221
|
hdel(key: string, itemId: string): this;
|
|
219
222
|
hget(key: string, itemId: string): this;
|
|
220
223
|
hgetall(key: string): this;
|
package/build/types/stream.d.ts
CHANGED
|
@@ -122,6 +122,8 @@ export type StreamConfig = {
|
|
|
122
122
|
guid: string;
|
|
123
123
|
/** Role associated with the stream */
|
|
124
124
|
role: StreamRole;
|
|
125
|
+
/** Default throttle (read from KeyType.THROTTLE_RATE) */
|
|
126
|
+
throttle: number;
|
|
125
127
|
/** Optional topic for the stream */
|
|
126
128
|
topic?: string;
|
|
127
129
|
/** Delay before a message can be reclaimed, defaults to 60,000 milliseconds */
|