@hotmeshio/hotmesh 0.0.36 → 0.0.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/build/modules/enums.d.ts +1 -0
- package/build/modules/enums.js +3 -1
- package/build/modules/errors.d.ts +9 -1
- package/build/modules/errors.js +12 -1
- package/build/modules/key.d.ts +20 -19
- package/build/modules/key.js +20 -20
- package/build/package.json +1 -1
- package/build/services/activities/activity.d.ts +10 -0
- package/build/services/activities/activity.js +28 -3
- package/build/services/activities/await.js +10 -9
- package/build/services/activities/cycle.js +10 -9
- package/build/services/activities/hook.d.ts +7 -1
- package/build/services/activities/hook.js +61 -44
- package/build/services/activities/interrupt.js +10 -9
- package/build/services/activities/signal.js +7 -7
- package/build/services/activities/trigger.js +4 -2
- package/build/services/activities/worker.js +9 -8
- package/build/services/durable/meshos.js +2 -2
- package/build/services/durable/worker.js +2 -2
- package/build/services/durable/workflow.js +17 -17
- package/build/services/engine/index.d.ts +5 -7
- package/build/services/engine/index.js +53 -47
- package/build/services/hotmesh/index.js +3 -3
- package/build/services/{signaler/stream.d.ts → router/index.d.ts} +3 -3
- package/build/services/{signaler/stream.js → router/index.js} +6 -6
- package/build/services/serializer/index.js +1 -1
- package/build/services/store/index.d.ts +9 -4
- package/build/services/store/index.js +21 -10
- package/build/services/task/index.d.ts +13 -4
- package/build/services/task/index.js +115 -17
- package/build/services/telemetry/index.js +6 -6
- package/build/services/worker/index.d.ts +3 -3
- package/build/services/worker/index.js +8 -8
- package/build/types/job.d.ts +2 -0
- package/build/types/stream.d.ts +1 -0
- package/modules/enums.ts +3 -0
- package/modules/errors.ts +18 -0
- package/modules/key.ts +21 -20
- package/package.json +1 -1
- package/services/activities/activity.ts +44 -4
- package/services/activities/await.ts +14 -10
- package/services/activities/cycle.ts +14 -10
- package/services/activities/hook.ts +70 -47
- package/services/activities/interrupt.ts +13 -10
- package/services/activities/signal.ts +11 -8
- package/services/activities/trigger.ts +5 -1
- package/services/activities/worker.ts +13 -9
- package/services/durable/meshos.ts +1 -1
- package/services/durable/worker.ts +1 -1
- package/services/durable/workflow.ts +1 -1
- package/services/engine/index.ts +82 -44
- package/services/hotmesh/index.ts +3 -3
- package/services/{signaler/stream.ts → router/index.ts} +5 -5
- package/services/serializer/index.ts +1 -1
- package/services/store/index.ts +23 -12
- package/services/task/index.ts +120 -21
- package/services/telemetry/index.ts +6 -6
- package/services/worker/index.ts +7 -7
- package/types/job.ts +2 -0
- package/types/stream.ts +6 -5
- package/build/services/signaler/store.d.ts +0 -15
- package/build/services/signaler/store.js +0 -68
- package/services/signaler/store.ts +0 -76
- /package/build/{services/durable/asyncLocalStorage.d.ts → modules/storage.d.ts} +0 -0
- /package/build/{services/durable/asyncLocalStorage.js → modules/storage.js} +0 -0
- /package/{services/durable/asyncLocalStorage.ts → modules/storage.ts} +0 -0
|
@@ -10,9 +10,8 @@ const utils_1 = require("../../modules/utils");
|
|
|
10
10
|
const activities_1 = __importDefault(require("../activities"));
|
|
11
11
|
const compiler_1 = require("../compiler");
|
|
12
12
|
const reporter_1 = require("../reporter");
|
|
13
|
+
const router_1 = require("../router");
|
|
13
14
|
const serializer_1 = require("../serializer");
|
|
14
|
-
const store_1 = require("../signaler/store");
|
|
15
|
-
const stream_1 = require("../signaler/stream");
|
|
16
15
|
const redis_1 = require("../store/clients/redis");
|
|
17
16
|
const ioredis_1 = require("../store/clients/ioredis");
|
|
18
17
|
const redis_2 = require("../stream/clients/redis");
|
|
@@ -20,7 +19,7 @@ const ioredis_2 = require("../stream/clients/ioredis");
|
|
|
20
19
|
const ioredis_3 = require("../sub/clients/ioredis");
|
|
21
20
|
const redis_3 = require("../sub/clients/redis");
|
|
22
21
|
const task_1 = require("../task");
|
|
23
|
-
const
|
|
22
|
+
const stream_1 = require("../../types/stream");
|
|
24
23
|
class EngineService {
|
|
25
24
|
constructor() {
|
|
26
25
|
this.cacheMode = 'cache';
|
|
@@ -40,13 +39,10 @@ class EngineService {
|
|
|
40
39
|
await instance.initStoreChannel(config.engine.store);
|
|
41
40
|
await instance.initSubChannel(config.engine.sub);
|
|
42
41
|
await instance.initStreamChannel(config.engine.stream);
|
|
43
|
-
instance.
|
|
44
|
-
instance.
|
|
45
|
-
//the storeSignaler service is used by the engine to create `webhooks`
|
|
46
|
-
//todo: unify/move to the task service (it manages all `signal` types)
|
|
47
|
-
instance.storeSignaler = new store_1.StoreSignaler(instance.store, logger);
|
|
42
|
+
instance.router = instance.initRouter(config);
|
|
43
|
+
instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
|
|
48
44
|
//the task service is used by the engine to process `webhooks` and `timehooks`
|
|
49
|
-
instance.
|
|
45
|
+
instance.taskService = new task_1.TaskService(instance.store, logger);
|
|
50
46
|
return instance;
|
|
51
47
|
}
|
|
52
48
|
}
|
|
@@ -84,12 +80,12 @@ class EngineService {
|
|
|
84
80
|
}
|
|
85
81
|
await this.stream.init(this.namespace, this.appId, this.logger);
|
|
86
82
|
}
|
|
87
|
-
|
|
88
|
-
return new
|
|
83
|
+
initRouter(config) {
|
|
84
|
+
return new router_1.Router({
|
|
89
85
|
namespace: this.namespace,
|
|
90
86
|
appId: this.appId,
|
|
91
87
|
guid: this.guid,
|
|
92
|
-
role:
|
|
88
|
+
role: stream_1.StreamRole.ENGINE,
|
|
93
89
|
reclaimDelay: config.engine.reclaimDelay,
|
|
94
90
|
reclaimCount: config.engine.reclaimCount,
|
|
95
91
|
}, this.stream, this.store, this.logger);
|
|
@@ -128,18 +124,17 @@ class EngineService {
|
|
|
128
124
|
}
|
|
129
125
|
}
|
|
130
126
|
async processWebHooks() {
|
|
131
|
-
this.
|
|
127
|
+
this.taskService.processWebHooks((this.hook).bind(this));
|
|
132
128
|
}
|
|
133
129
|
async processTimeHooks() {
|
|
134
|
-
this.
|
|
130
|
+
this.taskService.processTimeHooks((this.hookTime).bind(this));
|
|
135
131
|
}
|
|
136
132
|
async throttle(delayInMillis) {
|
|
137
|
-
this.
|
|
133
|
+
this.router.setThrottle(delayInMillis);
|
|
138
134
|
}
|
|
139
135
|
// ************* METADATA/MODEL METHODS *************
|
|
140
136
|
async initActivity(topic, data = {}, context) {
|
|
141
137
|
const [activityId, schema] = await this.getSchema(topic);
|
|
142
|
-
utils_1.polyfill;
|
|
143
138
|
const ActivityHandler = activities_1.default[utils_1.polyfill.resolveActivityType(schema.type)];
|
|
144
139
|
if (ActivityHandler) {
|
|
145
140
|
const utc = (0, utils_1.formatISODate)(new Date());
|
|
@@ -222,37 +217,43 @@ class EngineService {
|
|
|
222
217
|
async processStreamMessage(streamData) {
|
|
223
218
|
this.logger.debug('engine-process-stream-message', {
|
|
224
219
|
jid: streamData.metadata.jid,
|
|
220
|
+
gid: streamData.metadata.gid,
|
|
225
221
|
dad: streamData.metadata.dad,
|
|
226
222
|
aid: streamData.metadata.aid,
|
|
227
|
-
status: streamData.status ||
|
|
223
|
+
status: streamData.status || stream_1.StreamStatus.SUCCESS,
|
|
228
224
|
code: streamData.code || 200,
|
|
229
225
|
type: streamData.type,
|
|
230
226
|
});
|
|
231
227
|
const context = {
|
|
232
228
|
metadata: {
|
|
233
229
|
jid: streamData.metadata.jid,
|
|
230
|
+
gid: streamData.metadata.gid,
|
|
234
231
|
dad: streamData.metadata.dad,
|
|
235
232
|
aid: streamData.metadata.aid,
|
|
236
233
|
},
|
|
237
234
|
data: streamData.data,
|
|
238
235
|
};
|
|
239
|
-
if (streamData.type ===
|
|
236
|
+
if (streamData.type === stream_1.StreamDataType.TIMEHOOK) {
|
|
237
|
+
//TIMEHOOK AWAKEN
|
|
240
238
|
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
}
|
|
239
|
+
await activityHandler.processTimeHookEvent(streamData.metadata.jid);
|
|
240
|
+
}
|
|
241
|
+
else if (streamData.type === stream_1.StreamDataType.WEBHOOK) {
|
|
242
|
+
//WEBHOOK AWAKEN (SIGNAL IN)
|
|
243
|
+
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
|
|
244
|
+
await activityHandler.processWebHookEvent(streamData.status, streamData.code);
|
|
245
|
+
}
|
|
246
|
+
else if (streamData.type === stream_1.StreamDataType.TRANSITION) {
|
|
247
|
+
//TRANSITION (ADJACENT ACTIVITY)
|
|
248
|
+
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context); //todo: `as Activity` (type is more generic)
|
|
249
|
+
await activityHandler.process();
|
|
251
250
|
}
|
|
252
|
-
else if (streamData.type ===
|
|
251
|
+
else if (streamData.type === stream_1.StreamDataType.AWAIT) {
|
|
252
|
+
//TRIGGER JOB
|
|
253
253
|
context.metadata = {
|
|
254
254
|
...context.metadata,
|
|
255
255
|
pj: streamData.metadata.jid,
|
|
256
|
+
pg: streamData.metadata.gid,
|
|
256
257
|
pd: streamData.metadata.dad,
|
|
257
258
|
pa: streamData.metadata.aid,
|
|
258
259
|
trc: streamData.metadata.trc,
|
|
@@ -261,16 +262,19 @@ class EngineService {
|
|
|
261
262
|
const activityHandler = await this.initActivity(streamData.metadata.topic, streamData.data, context);
|
|
262
263
|
await activityHandler.process();
|
|
263
264
|
}
|
|
264
|
-
else if (streamData.type ===
|
|
265
|
+
else if (streamData.type === stream_1.StreamDataType.RESULT) {
|
|
266
|
+
//AWAIT RESULT
|
|
265
267
|
const activityHandler = await this.initActivity(`.${context.metadata.aid}`, streamData.data, context);
|
|
266
268
|
await activityHandler.processEvent(streamData.status, streamData.code);
|
|
267
269
|
}
|
|
268
270
|
else {
|
|
271
|
+
//WORKER RESULT
|
|
269
272
|
const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, streamData.data, context);
|
|
270
273
|
await activityHandler.processEvent(streamData.status, streamData.code, 'output');
|
|
271
274
|
}
|
|
272
275
|
this.logger.debug('engine-process-stream-message-end', {
|
|
273
276
|
jid: streamData.metadata.jid,
|
|
277
|
+
gid: streamData.metadata.gid,
|
|
274
278
|
aid: streamData.metadata.aid
|
|
275
279
|
});
|
|
276
280
|
}
|
|
@@ -284,28 +288,29 @@ class EngineService {
|
|
|
284
288
|
metadata: {
|
|
285
289
|
guid: (0, utils_1.guid)(),
|
|
286
290
|
jid: context.metadata.pj,
|
|
291
|
+
gid: context.metadata.pg,
|
|
287
292
|
dad: context.metadata.pd,
|
|
288
293
|
aid: context.metadata.pa,
|
|
289
294
|
trc: context.metadata.trc,
|
|
290
295
|
spn,
|
|
291
296
|
},
|
|
292
|
-
type:
|
|
297
|
+
type: stream_1.StreamDataType.RESULT,
|
|
293
298
|
data: jobOutput.data,
|
|
294
299
|
};
|
|
295
300
|
if (error && error.code) {
|
|
296
|
-
streamData.status =
|
|
301
|
+
streamData.status = stream_1.StreamStatus.ERROR;
|
|
297
302
|
streamData.data = error;
|
|
298
303
|
streamData.code = error.code;
|
|
299
304
|
}
|
|
300
305
|
else if (emit) {
|
|
301
|
-
streamData.status =
|
|
306
|
+
streamData.status = stream_1.StreamStatus.PENDING;
|
|
302
307
|
streamData.code = enums_1.STATUS_CODE_PENDING;
|
|
303
308
|
}
|
|
304
309
|
else {
|
|
305
|
-
streamData.status =
|
|
310
|
+
streamData.status = stream_1.StreamStatus.SUCCESS;
|
|
306
311
|
streamData.code = enums_1.STATUS_CODE_SUCCESS;
|
|
307
312
|
}
|
|
308
|
-
return (await this.
|
|
313
|
+
return (await this.router?.publishMessage(null, streamData));
|
|
309
314
|
}
|
|
310
315
|
}
|
|
311
316
|
hasParentJob(context) {
|
|
@@ -332,11 +337,11 @@ class EngineService {
|
|
|
332
337
|
await this.store.scrub(jobId);
|
|
333
338
|
}
|
|
334
339
|
// ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
|
|
335
|
-
async hook(topic, data, status =
|
|
336
|
-
const hookRule = await this.
|
|
340
|
+
async hook(topic, data, status = stream_1.StreamStatus.SUCCESS, code = 200) {
|
|
341
|
+
const hookRule = await this.taskService.getHookRule(topic);
|
|
337
342
|
const [aid] = await this.getSchema(`.${hookRule.to}`);
|
|
338
343
|
const streamData = {
|
|
339
|
-
type:
|
|
344
|
+
type: stream_1.StreamDataType.WEBHOOK,
|
|
340
345
|
status,
|
|
341
346
|
code,
|
|
342
347
|
metadata: {
|
|
@@ -346,9 +351,9 @@ class EngineService {
|
|
|
346
351
|
},
|
|
347
352
|
data,
|
|
348
353
|
};
|
|
349
|
-
return await this.
|
|
354
|
+
return await this.router.publishMessage(null, streamData);
|
|
350
355
|
}
|
|
351
|
-
async hookTime(jobId, activityId, type) {
|
|
356
|
+
async hookTime(jobId, gId, activityId, type) {
|
|
352
357
|
if (type === 'interrupt') {
|
|
353
358
|
return await this.interrupt(activityId, //note: 'activityId' is the actually job topic
|
|
354
359
|
jobId, { suppress: true, expire: 1 });
|
|
@@ -360,20 +365,21 @@ class EngineService {
|
|
|
360
365
|
const [aid, ...dimensions] = activityId.split(',');
|
|
361
366
|
const dad = `,${dimensions.join(',')}`;
|
|
362
367
|
const streamData = {
|
|
363
|
-
type:
|
|
368
|
+
type: stream_1.StreamDataType.TIMEHOOK,
|
|
364
369
|
metadata: {
|
|
365
370
|
guid: (0, utils_1.guid)(),
|
|
366
371
|
jid: jobId,
|
|
367
|
-
|
|
372
|
+
gid: gId,
|
|
368
373
|
dad,
|
|
374
|
+
aid,
|
|
369
375
|
},
|
|
370
376
|
data: { timestamp: Date.now() },
|
|
371
377
|
};
|
|
372
|
-
await this.
|
|
378
|
+
await this.router.publishMessage(null, streamData);
|
|
373
379
|
}
|
|
374
380
|
async hookAll(hookTopic, data, keyResolver, queryFacets = []) {
|
|
375
381
|
const config = await this.getVID();
|
|
376
|
-
const hookRule = await this.
|
|
382
|
+
const hookRule = await this.taskService.getHookRule(hookTopic);
|
|
377
383
|
if (hookRule) {
|
|
378
384
|
const subscriptionTopic = await (0, utils_1.getSubscriptionTopic)(hookRule.to, this.store, config);
|
|
379
385
|
const resolvedQuery = await this.resolveQuery(subscriptionTopic, keyResolver);
|
|
@@ -485,7 +491,7 @@ class EngineService {
|
|
|
485
491
|
}
|
|
486
492
|
}
|
|
487
493
|
async add(streamData) {
|
|
488
|
-
return await this.
|
|
494
|
+
return await this.router.publishMessage(null, streamData);
|
|
489
495
|
}
|
|
490
496
|
registerJobCallback(jobId, jobCallback) {
|
|
491
497
|
this.jobCallbacks[jobId] = jobCallback;
|
|
@@ -510,7 +516,7 @@ class EngineService {
|
|
|
510
516
|
this.pubPermSubs(context, jobOutput, options.emit);
|
|
511
517
|
}
|
|
512
518
|
if (!options.emit) {
|
|
513
|
-
this.
|
|
519
|
+
this.taskService.registerJobForCleanup(context.metadata.jid, this.resolveExpires(context, options), options);
|
|
514
520
|
}
|
|
515
521
|
return msgId;
|
|
516
522
|
}
|
|
@@ -520,7 +526,7 @@ class EngineService {
|
|
|
520
526
|
* it will be expired immediately.
|
|
521
527
|
*/
|
|
522
528
|
resolveExpires(context, options) {
|
|
523
|
-
return
|
|
529
|
+
return options.expire ?? context.metadata.expire ?? enums_1.DURABLE_EXPIRE_SECONDS;
|
|
524
530
|
}
|
|
525
531
|
// ****** GET JOB STATE/COLLATION STATUS BY ID *********
|
|
526
532
|
async getStatus(jobId) {
|
|
@@ -7,8 +7,8 @@ const redis_1 = require("../connector/clients/redis");
|
|
|
7
7
|
const ioredis_1 = require("../connector/clients/ioredis");
|
|
8
8
|
const engine_1 = require("../engine");
|
|
9
9
|
const logger_1 = require("../logger");
|
|
10
|
-
const stream_1 = require("../signaler/stream");
|
|
11
10
|
const quorum_1 = require("../quorum");
|
|
11
|
+
const router_1 = require("../router");
|
|
12
12
|
const worker_1 = require("../worker");
|
|
13
13
|
const connector_1 = require("../connector");
|
|
14
14
|
class HotMeshService {
|
|
@@ -141,13 +141,13 @@ class HotMeshService {
|
|
|
141
141
|
static async stop() {
|
|
142
142
|
if (!this.disconnecting) {
|
|
143
143
|
this.disconnecting = true;
|
|
144
|
-
await
|
|
144
|
+
await router_1.Router.stopConsuming();
|
|
145
145
|
await redis_1.RedisConnection.disconnectAll();
|
|
146
146
|
await ioredis_1.RedisConnection.disconnectAll();
|
|
147
147
|
}
|
|
148
148
|
}
|
|
149
149
|
stop() {
|
|
150
|
-
this.engine?.
|
|
150
|
+
this.engine?.taskService.cancelCleanup();
|
|
151
151
|
}
|
|
152
152
|
async compress(terms) {
|
|
153
153
|
return await this.engine?.compress(terms);
|
|
@@ -4,8 +4,8 @@ import { StoreService } from '../store';
|
|
|
4
4
|
import { StreamService } from '../stream';
|
|
5
5
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
6
6
|
import { ReclaimedMessageType, StreamConfig, StreamData, StreamDataResponse, StreamRole } from '../../types/stream';
|
|
7
|
-
declare class
|
|
8
|
-
static
|
|
7
|
+
declare class Router {
|
|
8
|
+
static instances: Set<Router>;
|
|
9
9
|
appId: string;
|
|
10
10
|
guid: string;
|
|
11
11
|
role: StreamRole;
|
|
@@ -40,4 +40,4 @@ declare class StreamSignaler {
|
|
|
40
40
|
expireUnacknowledged(reclaimedMessage: ReclaimedMessageType, stream: string, group: string, consumer: string, id: string, count: number): Promise<void>;
|
|
41
41
|
parseStreamData(str: string): [undefined, StreamData] | [Error];
|
|
42
42
|
}
|
|
43
|
-
export {
|
|
43
|
+
export { Router };
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.Router = void 0;
|
|
4
4
|
const enums_1 = require("../../modules/enums");
|
|
5
5
|
const key_1 = require("../../modules/key");
|
|
6
6
|
const utils_1 = require("../../modules/utils");
|
|
7
7
|
const telemetry_1 = require("../telemetry");
|
|
8
8
|
const stream_1 = require("../../types/stream");
|
|
9
|
-
class
|
|
9
|
+
class Router {
|
|
10
10
|
constructor(config, stream, store, logger) {
|
|
11
11
|
this.throttle = 0;
|
|
12
12
|
this.errorCount = 0;
|
|
@@ -35,7 +35,7 @@ class StreamSignaler {
|
|
|
35
35
|
}
|
|
36
36
|
async consumeMessages(stream, group, consumer, callback) {
|
|
37
37
|
this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
|
|
38
|
-
|
|
38
|
+
Router.instances.add(this);
|
|
39
39
|
this.shouldConsume = true;
|
|
40
40
|
await this.createGroup(stream, group);
|
|
41
41
|
let lastCheckedPendingMessagesAt = Date.now();
|
|
@@ -213,7 +213,7 @@ class StreamSignaler {
|
|
|
213
213
|
};
|
|
214
214
|
}
|
|
215
215
|
static async stopConsuming() {
|
|
216
|
-
for (const instance of [...
|
|
216
|
+
for (const instance of [...Router.instances]) {
|
|
217
217
|
instance.stopConsuming();
|
|
218
218
|
}
|
|
219
219
|
await (0, utils_1.sleepFor)(enums_1.BLOCK_TIME_MS);
|
|
@@ -311,5 +311,5 @@ class StreamSignaler {
|
|
|
311
311
|
}
|
|
312
312
|
}
|
|
313
313
|
}
|
|
314
|
-
exports.
|
|
315
|
-
|
|
314
|
+
exports.Router = Router;
|
|
315
|
+
Router.instances = new Set();
|
|
@@ -12,7 +12,7 @@ exports.MDATA_SYMBOLS = {
|
|
|
12
12
|
KEYS: ['au', 'err', 'l2s']
|
|
13
13
|
},
|
|
14
14
|
JOB: {
|
|
15
|
-
KEYS: ['ngn', 'tpc', 'pj', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
|
|
15
|
+
KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
|
|
16
16
|
},
|
|
17
17
|
JOB_UPDATE: {
|
|
18
18
|
KEYS: ['ju', 'err']
|
|
@@ -39,7 +39,12 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
39
39
|
zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null>;
|
|
40
40
|
mintKey(type: KeyType, params: KeyStoreParams): string;
|
|
41
41
|
invalidateCache(): void;
|
|
42
|
-
|
|
42
|
+
/**
|
|
43
|
+
* At any given time only a single engine will
|
|
44
|
+
* check for and process work items in the
|
|
45
|
+
* time and signal task queues.
|
|
46
|
+
*/
|
|
47
|
+
reserveScoutRole(scoutType: 'time' | 'signal', delay?: number): Promise<boolean>;
|
|
43
48
|
getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
|
|
44
49
|
setSettings(manifest: HotMeshSettings): Promise<any>;
|
|
45
50
|
reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;
|
|
@@ -61,7 +66,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
61
66
|
* list (added via RPUSH) are LPOPed. If origin was expired, then
|
|
62
67
|
* LPOPed items from the list are likewise expired;
|
|
63
68
|
*/
|
|
64
|
-
setDependency(originJobId: string, topic: string, jobId: string, multi?: U): Promise<any>;
|
|
69
|
+
setDependency(originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
|
|
65
70
|
setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
|
|
66
71
|
hGetAllResult(result: any): any;
|
|
67
72
|
getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
|
|
@@ -107,8 +112,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
|
|
|
107
112
|
* for the given sleep group. Sleep groups are
|
|
108
113
|
* organized into 'n'-second blocks (LISTS))
|
|
109
114
|
*/
|
|
110
|
-
registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', deletionTime: number, multi?: U): Promise<void>;
|
|
111
|
-
getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt'] |
|
|
115
|
+
registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', deletionTime: number, multi?: U): Promise<void>;
|
|
116
|
+
getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt'] | boolean>;
|
|
112
117
|
/**
|
|
113
118
|
* when processing time jobs, the target LIST ID returned
|
|
114
119
|
* from the ZSET query can be prefixed to denote what to
|
|
@@ -99,10 +99,19 @@ class StoreService {
|
|
|
99
99
|
invalidateCache() {
|
|
100
100
|
this.cache.invalidate();
|
|
101
101
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
102
|
+
/**
|
|
103
|
+
* At any given time only a single engine will
|
|
104
|
+
* check for and process work items in the
|
|
105
|
+
* time and signal task queues.
|
|
106
|
+
*/
|
|
107
|
+
async reserveScoutRole(scoutType, delay = enums_1.SCOUT_INTERVAL_SECONDS) {
|
|
108
|
+
const key = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
|
|
109
|
+
const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`);
|
|
110
|
+
if (this.isSuccessful(success)) {
|
|
111
|
+
await this.redisClient[this.commands.expire](key, delay - 1);
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
return false;
|
|
106
115
|
}
|
|
107
116
|
async getSettings(bCreate = false) {
|
|
108
117
|
let settings = this.cache?.getSettings();
|
|
@@ -304,11 +313,11 @@ class StoreService {
|
|
|
304
313
|
* list (added via RPUSH) are LPOPed. If origin was expired, then
|
|
305
314
|
* LPOPed items from the list are likewise expired;
|
|
306
315
|
*/
|
|
307
|
-
async setDependency(originJobId, topic, jobId, multi) {
|
|
316
|
+
async setDependency(originJobId, topic, jobId, gId, multi) {
|
|
308
317
|
const privateMulti = multi || this.getMulti();
|
|
309
318
|
const depParams = { appId: this.appId, jobId: originJobId };
|
|
310
319
|
const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
|
|
311
|
-
privateMulti[this.commands.rpush](depKey, `expire::${topic}::${jobId}`);
|
|
320
|
+
privateMulti[this.commands.rpush](depKey, `expire::${topic}::${gId}::${jobId}`);
|
|
312
321
|
if (!multi) {
|
|
313
322
|
return await privateMulti.exec();
|
|
314
323
|
}
|
|
@@ -682,9 +691,9 @@ class StoreService {
|
|
|
682
691
|
* for the given sleep group. Sleep groups are
|
|
683
692
|
* organized into 'n'-second blocks (LISTS))
|
|
684
693
|
*/
|
|
685
|
-
async registerTimeHook(jobId, activityId, type, deletionTime, multi) {
|
|
694
|
+
async registerTimeHook(jobId, gId, activityId, type, deletionTime, multi) {
|
|
686
695
|
const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
|
|
687
|
-
const timeEvent = `${type}::${activityId}::${jobId}`;
|
|
696
|
+
const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
|
|
688
697
|
const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
|
|
689
698
|
if (multi || len === 1) {
|
|
690
699
|
const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
|
|
@@ -692,6 +701,7 @@ class StoreService {
|
|
|
692
701
|
}
|
|
693
702
|
}
|
|
694
703
|
async getNextTimeJob(listKey) {
|
|
704
|
+
const existing = Boolean(listKey);
|
|
695
705
|
const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
|
|
696
706
|
listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
|
|
697
707
|
if (listKey) {
|
|
@@ -699,11 +709,12 @@ class StoreService {
|
|
|
699
709
|
const timeEvent = await this.redisClient[this.commands.lpop](pKey);
|
|
700
710
|
if (timeEvent) {
|
|
701
711
|
//there are 3 time-related event triggers: sleep, expire, interrupt
|
|
702
|
-
const [_type, activityId, ...jobId] = timeEvent.split('::');
|
|
703
|
-
return [listKey, jobId.join('::'), activityId, pType];
|
|
712
|
+
const [_type, activityId, gId, ...jobId] = timeEvent.split('::');
|
|
713
|
+
return [listKey, jobId.join('::'), gId, activityId, pType];
|
|
704
714
|
}
|
|
705
715
|
await this.redisClient[this.commands.zrem](zsetKey, listKey);
|
|
706
716
|
}
|
|
717
|
+
return existing;
|
|
707
718
|
}
|
|
708
719
|
/**
|
|
709
720
|
* when processing time jobs, the target LIST ID returned
|
|
@@ -1,19 +1,28 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { ILogger } from '../logger';
|
|
3
3
|
import { StoreService } from '../store';
|
|
4
|
-
import { HookInterface } from '../../types/hook';
|
|
5
|
-
import { JobCompletionOptions } from '../../types/job';
|
|
4
|
+
import { HookInterface, HookRule } from '../../types/hook';
|
|
5
|
+
import { JobCompletionOptions, JobState } from '../../types/job';
|
|
6
6
|
import { RedisClient, RedisMulti } from '../../types/redis';
|
|
7
7
|
declare class TaskService {
|
|
8
8
|
store: StoreService<RedisClient, RedisMulti>;
|
|
9
9
|
logger: ILogger;
|
|
10
10
|
cleanupTimeout: NodeJS.Timeout | null;
|
|
11
|
+
isScout: boolean;
|
|
11
12
|
constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
|
|
12
13
|
processWebHooks(hookEventCallback: HookInterface): Promise<void>;
|
|
13
14
|
enqueueWorkItems(keys: string[]): Promise<void>;
|
|
14
15
|
registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
|
|
15
|
-
registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', inSeconds?: number, multi?: RedisMulti): Promise<void>;
|
|
16
|
-
|
|
16
|
+
registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', inSeconds?: number, multi?: RedisMulti): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Should this engine instance play the role of 'scout' for the quorum.
|
|
19
|
+
*/
|
|
20
|
+
shouldScout(): Promise<boolean>;
|
|
21
|
+
processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt') => Promise<void>, listKey?: string): Promise<void>;
|
|
17
22
|
cancelCleanup(): void;
|
|
23
|
+
getHookRule(topic: string): Promise<HookRule | undefined>;
|
|
24
|
+
registerWebHook(topic: string, context: JobState, dad: string, multi?: RedisMulti): Promise<string>;
|
|
25
|
+
processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<[string, string, string, string] | undefined>;
|
|
26
|
+
deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
|
|
18
27
|
}
|
|
19
28
|
export { TaskService };
|