@hotmeshio/hotmesh 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/build/modules/errors.d.ts +22 -1
- package/build/modules/errors.js +28 -1
- package/build/modules/utils.d.ts +2 -1
- package/build/modules/utils.js +5 -1
- package/build/package.json +7 -2
- package/build/services/activities/activity.d.ts +2 -0
- package/build/services/activities/activity.js +16 -10
- package/build/services/activities/await.d.ts +2 -6
- package/build/services/activities/await.js +12 -75
- package/build/services/activities/cycle.js +2 -2
- package/build/services/activities/index.d.ts +2 -2
- package/build/services/activities/index.js +2 -2
- package/build/services/activities/signal.d.ts +16 -0
- package/build/services/activities/signal.js +94 -0
- package/build/services/activities/trigger.js +4 -3
- package/build/services/activities/worker.d.ts +2 -1
- package/build/services/activities/worker.js +11 -6
- package/build/services/compiler/deployer.js +3 -1
- package/build/services/durable/client.d.ts +4 -3
- package/build/services/durable/client.js +48 -21
- package/build/services/durable/factory.d.ts +22 -18
- package/build/services/durable/factory.js +722 -50
- package/build/services/durable/handle.d.ts +1 -0
- package/build/services/durable/handle.js +5 -1
- package/build/services/durable/worker.d.ts +3 -8
- package/build/services/durable/worker.js +75 -73
- package/build/services/durable/workflow.d.ts +5 -0
- package/build/services/durable/workflow.js +93 -24
- package/build/services/engine/index.d.ts +5 -5
- package/build/services/engine/index.js +24 -14
- package/build/services/hotmesh/index.d.ts +1 -0
- package/build/services/hotmesh/index.js +3 -1
- package/build/services/mapper/index.js +1 -1
- package/build/services/pipe/functions/array.d.ts +1 -0
- package/build/services/pipe/functions/array.js +3 -0
- package/build/services/reporter/index.js +9 -2
- package/build/services/signaler/store.js +8 -3
- package/build/services/signaler/stream.js +3 -3
- package/build/services/store/clients/ioredis.js +15 -15
- package/build/services/store/clients/redis.js +18 -18
- package/build/services/store/index.d.ts +1 -1
- package/build/services/store/index.js +11 -3
- package/build/services/task/index.js +3 -3
- package/build/types/activity.d.ts +15 -6
- package/build/types/durable.d.ts +15 -2
- package/build/types/index.d.ts +1 -1
- package/build/types/stats.d.ts +1 -0
- package/modules/errors.ts +35 -0
- package/modules/utils.ts +5 -1
- package/package.json +7 -2
- package/services/activities/activity.ts +19 -9
- package/services/activities/await.ts +14 -90
- package/services/activities/cycle.ts +2 -2
- package/services/activities/index.ts +2 -2
- package/services/activities/signal.ts +124 -0
- package/services/activities/trigger.ts +4 -3
- package/services/activities/worker.ts +13 -13
- package/services/compiler/deployer.ts +3 -1
- package/services/durable/client.ts +60 -22
- package/services/durable/factory.ts +723 -49
- package/services/durable/handle.ts +6 -1
- package/services/durable/worker.ts +92 -79
- package/services/durable/workflow.ts +95 -25
- package/services/engine/index.ts +31 -22
- package/services/hotmesh/index.ts +6 -3
- package/services/mapper/index.ts +1 -1
- package/services/pipe/functions/array.ts +4 -0
- package/services/reporter/index.ts +10 -2
- package/services/signaler/store.ts +8 -3
- package/services/signaler/stream.ts +3 -3
- package/services/store/clients/ioredis.ts +15 -15
- package/services/store/clients/redis.ts +18 -18
- package/services/store/index.ts +12 -3
- package/services/task/index.ts +3 -3
- package/types/activity.ts +16 -7
- package/types/durable.ts +17 -1
- package/types/index.ts +1 -1
- package/types/stats.ts +1 -0
- package/build/services/activities/emit.d.ts +0 -9
- package/build/services/activities/emit.js +0 -13
- package/services/activities/emit.ts +0 -25
package/README.md
CHANGED
|
@@ -161,10 +161,10 @@ const jobId = await hotMesh.pub(topic, payload);
|
|
|
161
161
|
```
|
|
162
162
|
|
|
163
163
|
### Subscribe to Events
|
|
164
|
-
Call `
|
|
164
|
+
Call `psub` (patterned subscription) to subscribe to all workflow results for a given topic.
|
|
165
165
|
|
|
166
166
|
```javascript
|
|
167
|
-
await hotMesh.
|
|
167
|
+
await hotMesh.psub('sandbox.work.done.*', (topic, jobOutput) => {
|
|
168
168
|
// use jobOutput.data
|
|
169
169
|
});
|
|
170
170
|
```
|
|
@@ -6,6 +6,27 @@ declare class GetStateError extends Error {
|
|
|
6
6
|
declare class SetStateError extends Error {
|
|
7
7
|
constructor();
|
|
8
8
|
}
|
|
9
|
+
declare class DurableIncompleteSignalError extends Error {
|
|
10
|
+
code: number;
|
|
11
|
+
constructor(message: string);
|
|
12
|
+
}
|
|
13
|
+
declare class DurableWaitForSignalError extends Error {
|
|
14
|
+
code: number;
|
|
15
|
+
signals: {
|
|
16
|
+
signal: string;
|
|
17
|
+
index: number;
|
|
18
|
+
}[];
|
|
19
|
+
constructor(message: string, signals: {
|
|
20
|
+
signal: string;
|
|
21
|
+
index: number;
|
|
22
|
+
}[]);
|
|
23
|
+
}
|
|
24
|
+
declare class DurableSleepError extends Error {
|
|
25
|
+
code: number;
|
|
26
|
+
duration: number;
|
|
27
|
+
index: number;
|
|
28
|
+
constructor(message: string, duration: number, index: number);
|
|
29
|
+
}
|
|
9
30
|
declare class DurableTimeoutError extends Error {
|
|
10
31
|
code: number;
|
|
11
32
|
constructor(message: string);
|
|
@@ -41,4 +62,4 @@ declare class CollationError extends Error {
|
|
|
41
62
|
fault: CollationFaultType;
|
|
42
63
|
constructor(status: number, leg: ActivityDuplex, stage: CollationStage, fault?: CollationFaultType);
|
|
43
64
|
}
|
|
44
|
-
export { CollationError, DurableTimeoutError, DurableMaxedError, DurableFatalError, DurableRetryError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };
|
|
65
|
+
export { CollationError, DurableTimeoutError, DurableMaxedError, DurableFatalError, DurableRetryError, DurableWaitForSignalError, DurableIncompleteSignalError, DurableSleepError, DuplicateJobError, GetStateError, SetStateError, MapDataError, RegisterTimeoutError, ExecActivityError };
|
package/build/modules/errors.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ExecActivityError = exports.RegisterTimeoutError = exports.MapDataError = exports.SetStateError = exports.GetStateError = exports.DuplicateJobError = exports.DurableRetryError = exports.DurableFatalError = exports.DurableMaxedError = exports.DurableTimeoutError = exports.CollationError = void 0;
|
|
3
|
+
exports.ExecActivityError = exports.RegisterTimeoutError = exports.MapDataError = exports.SetStateError = exports.GetStateError = exports.DuplicateJobError = exports.DurableSleepError = exports.DurableIncompleteSignalError = exports.DurableWaitForSignalError = exports.DurableRetryError = exports.DurableFatalError = exports.DurableMaxedError = exports.DurableTimeoutError = exports.CollationError = void 0;
|
|
4
4
|
class GetStateError extends Error {
|
|
5
5
|
constructor() {
|
|
6
6
|
super("Error occurred while getting job state");
|
|
@@ -13,6 +13,33 @@ class SetStateError extends Error {
|
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
15
|
exports.SetStateError = SetStateError;
|
|
16
|
+
//thrown when a signal set is incomplete but already configured
|
|
17
|
+
//if a waitFor set has 'n' items, this can be thrown `n - 1` times
|
|
18
|
+
class DurableIncompleteSignalError extends Error {
|
|
19
|
+
constructor(message) {
|
|
20
|
+
super(message);
|
|
21
|
+
this.code = 593;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
exports.DurableIncompleteSignalError = DurableIncompleteSignalError;
|
|
25
|
+
//the original waitFor error that is thrown for a new signal set
|
|
26
|
+
class DurableWaitForSignalError extends Error {
|
|
27
|
+
constructor(message, signals) {
|
|
28
|
+
super(message);
|
|
29
|
+
this.signals = signals;
|
|
30
|
+
this.code = 594;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
exports.DurableWaitForSignalError = DurableWaitForSignalError;
|
|
34
|
+
class DurableSleepError extends Error {
|
|
35
|
+
constructor(message, duration, index) {
|
|
36
|
+
super(message);
|
|
37
|
+
this.duration = duration;
|
|
38
|
+
this.index = index;
|
|
39
|
+
this.code = 595;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
exports.DurableSleepError = DurableSleepError;
|
|
16
43
|
class DurableTimeoutError extends Error {
|
|
17
44
|
constructor(message) {
|
|
18
45
|
super(message);
|
package/build/modules/utils.d.ts
CHANGED
|
@@ -21,7 +21,8 @@ export declare function findSubscriptionForTrigger(obj: AppSubscriptions, value:
|
|
|
21
21
|
*/
|
|
22
22
|
export declare function getSubscriptionTopic(activityId: string, store: StoreService<RedisClient, RedisMulti>, appVID: AppVID): Promise<string | undefined>;
|
|
23
23
|
/**
|
|
24
|
-
* returns the 12-digit format of the iso timestamp (e.g, 202101010000)
|
|
24
|
+
* returns the 12-digit format of the iso timestamp (e.g, 202101010000); returns
|
|
25
|
+
* an empty string if overridden by the user to not segment by time (infinity).
|
|
25
26
|
*/
|
|
26
27
|
export declare function getTimeSeries(granularity: string): string;
|
|
27
28
|
export declare function formatISODate(input: Date | string): string;
|
package/build/modules/utils.js
CHANGED
|
@@ -85,9 +85,13 @@ async function getSubscriptionTopic(activityId, store, appVID) {
|
|
|
85
85
|
}
|
|
86
86
|
exports.getSubscriptionTopic = getSubscriptionTopic;
|
|
87
87
|
/**
|
|
88
|
-
* returns the 12-digit format of the iso timestamp (e.g, 202101010000)
|
|
88
|
+
* returns the 12-digit format of the iso timestamp (e.g, 202101010000); returns
|
|
89
|
+
* an empty string if overridden by the user to not segment by time (infinity).
|
|
89
90
|
*/
|
|
90
91
|
function getTimeSeries(granularity) {
|
|
92
|
+
if (granularity.toString() === 'infinity') {
|
|
93
|
+
return '0';
|
|
94
|
+
}
|
|
91
95
|
const now = new Date();
|
|
92
96
|
const granularityUnit = granularity.slice(-1);
|
|
93
97
|
const granularityValue = parseInt(granularity.slice(0, -1), 10);
|
package/build/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hotmeshio/hotmesh",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.0.13",
|
|
4
|
+
"description": "Unbreakable Workflows",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"types": "build/index.d.ts",
|
|
7
7
|
"repository": {
|
|
@@ -26,6 +26,9 @@
|
|
|
26
26
|
"test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
27
27
|
"test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
28
28
|
"test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
|
|
29
|
+
"test:emit": "NODE_ENV=test jest ./tests/functional/emit/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
30
|
+
"test:hook": "NODE_ENV=test jest ./tests/functional/hook/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
31
|
+
"test:signal": "NODE_ENV=test jest ./tests/functional/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
29
32
|
"test:parallel": "NODE_ENV=test jest ./tests/functional/parallel/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
30
33
|
"test:sequence": "NODE_ENV=test jest ./tests/functional/sequence/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
31
34
|
"test:quorum": "NODE_ENV=test jest ./tests/functional/quorum/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
@@ -44,6 +47,8 @@
|
|
|
44
47
|
"test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
45
48
|
"test:durable:retry": "NODE_ENV=test jest ./tests/durable/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
46
49
|
"test:durable:fatal": "NODE_ENV=test jest ./tests/durable/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
50
|
+
"test:durable:sleep": "NODE_ENV=test jest ./tests/durable/sleep/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
51
|
+
"test:durable:signal": "NODE_ENV=test jest ./tests/durable/signal/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
47
52
|
"test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
|
|
48
53
|
"test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
|
|
49
54
|
},
|
|
@@ -7,6 +7,7 @@ import { JobState, JobStatus } from '../../types/job';
|
|
|
7
7
|
import { MultiResponseFlags, RedisClient, RedisMulti } from '../../types/redis';
|
|
8
8
|
import { StringAnyType } from '../../types/serializer';
|
|
9
9
|
import { StreamCode, StreamData, StreamStatus } from '../../types/stream';
|
|
10
|
+
import { HookRule } from '../../types/hook';
|
|
10
11
|
/**
|
|
11
12
|
* The base class for all activities
|
|
12
13
|
*/
|
|
@@ -28,6 +29,7 @@ declare class Activity {
|
|
|
28
29
|
process(): Promise<string>;
|
|
29
30
|
setLeg(leg: ActivityLeg): void;
|
|
30
31
|
doesHook(): boolean;
|
|
32
|
+
getHookRule(topic: string): Promise<HookRule | undefined>;
|
|
31
33
|
registerHook(multi?: RedisMulti): Promise<string | void>;
|
|
32
34
|
processWebHookEvent(): Promise<JobStatus | void>;
|
|
33
35
|
processTimeHookEvent(jobId: string): Promise<JobStatus | void>;
|
|
@@ -72,10 +72,10 @@ class Activity {
|
|
|
72
72
|
}
|
|
73
73
|
catch (error) {
|
|
74
74
|
if (error instanceof errors_1.GetStateError) {
|
|
75
|
-
this.logger.error('activity-get-state-error', error);
|
|
75
|
+
this.logger.error('activity-get-state-error', { error });
|
|
76
76
|
}
|
|
77
77
|
else {
|
|
78
|
-
this.logger.error('activity-process-error', error);
|
|
78
|
+
this.logger.error('activity-process-error', { error });
|
|
79
79
|
}
|
|
80
80
|
telemetry.setActivityError(error.message);
|
|
81
81
|
throw error;
|
|
@@ -92,6 +92,10 @@ class Activity {
|
|
|
92
92
|
doesHook() {
|
|
93
93
|
return !!(this.config.hook?.topic || this.config.sleep);
|
|
94
94
|
}
|
|
95
|
+
async getHookRule(topic) {
|
|
96
|
+
const rules = await this.store.getHookRules();
|
|
97
|
+
return rules?.[topic]?.[0];
|
|
98
|
+
}
|
|
95
99
|
async registerHook(multi) {
|
|
96
100
|
if (this.config.hook?.topic) {
|
|
97
101
|
const signaler = new store_1.StoreSignaler(this.store, this.logger);
|
|
@@ -155,7 +159,7 @@ class Activity {
|
|
|
155
159
|
return jobStatus;
|
|
156
160
|
}
|
|
157
161
|
catch (error) {
|
|
158
|
-
this.logger.error('engine-process-hook-event-error', error);
|
|
162
|
+
this.logger.error('engine-process-hook-event-error', { error });
|
|
159
163
|
telemetry.setActivityError(error.message);
|
|
160
164
|
throw error;
|
|
161
165
|
}
|
|
@@ -197,12 +201,12 @@ class Activity {
|
|
|
197
201
|
this.transitionAdjacent(multiResponse, telemetry);
|
|
198
202
|
}
|
|
199
203
|
catch (error) {
|
|
200
|
-
this.logger.error('activity-process-event-error', error);
|
|
201
|
-
telemetry.setActivityError(error.message);
|
|
204
|
+
this.logger.error('activity-process-event-error', { error });
|
|
205
|
+
telemetry && telemetry.setActivityError(error.message);
|
|
202
206
|
throw error;
|
|
203
207
|
}
|
|
204
208
|
finally {
|
|
205
|
-
telemetry.endActivitySpan();
|
|
209
|
+
telemetry && telemetry.endActivitySpan();
|
|
206
210
|
this.logger.debug('activity-process-event-end', { jid, aid });
|
|
207
211
|
}
|
|
208
212
|
}
|
|
@@ -484,16 +488,18 @@ class Activity {
|
|
|
484
488
|
}
|
|
485
489
|
async transition(adjacencyList, jobStatus) {
|
|
486
490
|
let mIds = [];
|
|
487
|
-
if (
|
|
491
|
+
if (jobStatus <= 0 || this.config.emit) {
|
|
492
|
+
//activity should not send 'emit' if the job is truly over
|
|
493
|
+
const isTrueEmit = jobStatus > 0;
|
|
494
|
+
await this.engine.runJobCompletionTasks(this.context, isTrueEmit);
|
|
495
|
+
}
|
|
496
|
+
if (adjacencyList.length && jobStatus > 0) {
|
|
488
497
|
const multi = this.store.getMulti();
|
|
489
498
|
for (const execSignal of adjacencyList) {
|
|
490
499
|
await this.engine.streamSignaler?.publishMessage(null, execSignal, multi);
|
|
491
500
|
}
|
|
492
501
|
mIds = (await multi.exec());
|
|
493
502
|
}
|
|
494
|
-
else if (jobStatus <= 0) {
|
|
495
|
-
await this.engine.runJobCompletionTasks(this.context);
|
|
496
|
-
}
|
|
497
503
|
return mIds;
|
|
498
504
|
}
|
|
499
505
|
}
|
|
@@ -2,15 +2,11 @@ import { Activity } from './activity';
|
|
|
2
2
|
import { EngineService } from '../engine';
|
|
3
3
|
import { ActivityData, ActivityMetadata, AwaitActivity, ActivityType } from '../../types/activity';
|
|
4
4
|
import { JobState } from '../../types/job';
|
|
5
|
-
import {
|
|
6
|
-
import { StreamCode, StreamData, StreamStatus } from '../../types/stream';
|
|
5
|
+
import { RedisMulti } from '../../types/redis';
|
|
7
6
|
declare class Await extends Activity {
|
|
8
7
|
config: AwaitActivity;
|
|
9
8
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
10
9
|
process(): Promise<string>;
|
|
11
|
-
execActivity(): Promise<string>;
|
|
12
|
-
processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
|
|
13
|
-
processSuccessResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
14
|
-
processErrorResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags>;
|
|
10
|
+
execActivity(multi: RedisMulti): Promise<string>;
|
|
15
11
|
}
|
|
16
12
|
export { Await };
|
|
@@ -3,9 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Await = void 0;
|
|
4
4
|
const errors_1 = require("../../modules/errors");
|
|
5
5
|
const activity_1 = require("./activity");
|
|
6
|
+
const collator_1 = require("../collator");
|
|
6
7
|
const stream_1 = require("../../types/stream");
|
|
7
8
|
const telemetry_1 = require("../telemetry");
|
|
8
|
-
const
|
|
9
|
+
const pipe_1 = require("../pipe");
|
|
9
10
|
class Await extends activity_1.Activity {
|
|
10
11
|
constructor(config, data, metadata, hook, engine, context) {
|
|
11
12
|
super(config, data, metadata, hook, engine, context);
|
|
@@ -15,21 +16,24 @@ class Await extends activity_1.Activity {
|
|
|
15
16
|
this.logger.debug('await-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
16
17
|
let telemetry;
|
|
17
18
|
try {
|
|
19
|
+
//confirm entry is allowed and restore state
|
|
18
20
|
this.setLeg(1);
|
|
19
21
|
await collator_1.CollatorService.notarizeEntry(this);
|
|
20
22
|
await this.getState();
|
|
21
23
|
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
22
24
|
telemetry.startActivitySpan(this.leg);
|
|
23
25
|
this.mapInputData();
|
|
26
|
+
//save state and authorize reentry
|
|
24
27
|
const multi = this.store.getMulti();
|
|
25
28
|
//todo: await this.registerTimeout();
|
|
29
|
+
const messageId = await this.execActivity(multi);
|
|
26
30
|
await collator_1.CollatorService.authorizeReentry(this, multi);
|
|
27
31
|
await this.setState(multi);
|
|
28
32
|
await this.setStatus(0, multi);
|
|
29
33
|
const multiResponse = await multi.exec();
|
|
34
|
+
//telemetry
|
|
30
35
|
telemetry.mapActivityAttributes();
|
|
31
36
|
const jobStatus = this.resolveStatus(multiResponse);
|
|
32
|
-
const messageId = await this.execActivity();
|
|
33
37
|
telemetry.setActivityAttributes({
|
|
34
38
|
'app.activity.mid': messageId,
|
|
35
39
|
'app.job.jss': jobStatus
|
|
@@ -39,10 +43,10 @@ class Await extends activity_1.Activity {
|
|
|
39
43
|
catch (error) {
|
|
40
44
|
telemetry.setActivityError(error.message);
|
|
41
45
|
if (error instanceof errors_1.GetStateError) {
|
|
42
|
-
this.logger.error('await-get-state-error', error);
|
|
46
|
+
this.logger.error('await-get-state-error', { error });
|
|
43
47
|
}
|
|
44
48
|
else {
|
|
45
|
-
this.logger.error('await-process-error', error);
|
|
49
|
+
this.logger.error('await-process-error', { error });
|
|
46
50
|
}
|
|
47
51
|
throw error;
|
|
48
52
|
}
|
|
@@ -51,13 +55,14 @@ class Await extends activity_1.Activity {
|
|
|
51
55
|
this.logger.debug('await-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
|
-
async execActivity() {
|
|
58
|
+
async execActivity(multi) {
|
|
59
|
+
const topic = pipe_1.Pipe.resolve(this.config.subtype, this.context);
|
|
55
60
|
const streamData = {
|
|
56
61
|
metadata: {
|
|
57
62
|
jid: this.context.metadata.jid,
|
|
58
63
|
dad: this.metadata.dad,
|
|
59
64
|
aid: this.metadata.aid,
|
|
60
|
-
topic
|
|
65
|
+
topic,
|
|
61
66
|
spn: this.context['$self'].output.metadata?.l1s,
|
|
62
67
|
trc: this.context.metadata.trc,
|
|
63
68
|
},
|
|
@@ -69,75 +74,7 @@ class Await extends activity_1.Activity {
|
|
|
69
74
|
retry: this.config.retry
|
|
70
75
|
};
|
|
71
76
|
}
|
|
72
|
-
return (await this.engine.streamSignaler?.publishMessage(null, streamData));
|
|
73
|
-
}
|
|
74
|
-
//******** `RESOLVE` ENTRY POINT (B) ********//
|
|
75
|
-
//this method is invoked when the job spawned by this job ends;
|
|
76
|
-
//`this.data` is the job data produced by the spawned job
|
|
77
|
-
async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200) {
|
|
78
|
-
this.setLeg(2);
|
|
79
|
-
const jid = this.context.metadata.jid;
|
|
80
|
-
const aid = this.metadata.aid;
|
|
81
|
-
if (!jid) {
|
|
82
|
-
throw new Error('await-process-event-error');
|
|
83
|
-
}
|
|
84
|
-
this.logger.debug('await-resolve-await', { jid, aid, status, code });
|
|
85
|
-
this.status = status;
|
|
86
|
-
this.code = code;
|
|
87
|
-
let telemetry;
|
|
88
|
-
try {
|
|
89
|
-
await this.getState();
|
|
90
|
-
const aState = await collator_1.CollatorService.notarizeReentry(this);
|
|
91
|
-
this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(aState);
|
|
92
|
-
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
93
|
-
telemetry.startActivitySpan(this.leg);
|
|
94
|
-
let multiResponse = [];
|
|
95
|
-
if (status === stream_1.StreamStatus.SUCCESS) {
|
|
96
|
-
this.bindActivityData('output');
|
|
97
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
98
|
-
multiResponse = await this.processSuccessResponse(this.adjacencyList);
|
|
99
|
-
}
|
|
100
|
-
else {
|
|
101
|
-
this.bindActivityError(this.data);
|
|
102
|
-
this.adjacencyList = await this.filterAdjacent();
|
|
103
|
-
multiResponse = await this.processErrorResponse(this.adjacencyList);
|
|
104
|
-
}
|
|
105
|
-
telemetry.mapActivityAttributes();
|
|
106
|
-
const jobStatus = this.resolveStatus(multiResponse);
|
|
107
|
-
const attrs = { 'app.job.jss': jobStatus };
|
|
108
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
109
|
-
if (messageIds.length) {
|
|
110
|
-
attrs['app.activity.mids'] = messageIds.join(',');
|
|
111
|
-
}
|
|
112
|
-
telemetry.setActivityAttributes(attrs);
|
|
113
|
-
}
|
|
114
|
-
catch (error) {
|
|
115
|
-
this.logger.error('await-resolve-await-error', error);
|
|
116
|
-
telemetry.setActivityError(error.message);
|
|
117
|
-
throw error;
|
|
118
|
-
}
|
|
119
|
-
finally {
|
|
120
|
-
telemetry.endActivitySpan();
|
|
121
|
-
this.logger.debug('await-resolve-await-end', { jid, aid, status, code });
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
async processSuccessResponse(adjacencyList) {
|
|
125
|
-
this.mapJobData();
|
|
126
|
-
const multi = this.store.getMulti();
|
|
127
|
-
await this.setState(multi);
|
|
128
|
-
await collator_1.CollatorService.notarizeCompletion(this, multi);
|
|
129
|
-
await this.setStatus(adjacencyList.length - 1, multi);
|
|
130
|
-
return await multi.exec();
|
|
131
|
-
}
|
|
132
|
-
async processErrorResponse(adjacencyList) {
|
|
133
|
-
//todo: if adjacencyList.length == 0, then map to the job output
|
|
134
|
-
// this method would be added to Base activity class
|
|
135
|
-
//this.mapJobData();
|
|
136
|
-
const multi = this.store.getMulti();
|
|
137
|
-
await this.setState(multi);
|
|
138
|
-
await collator_1.CollatorService.notarizeCompletion(this, multi);
|
|
139
|
-
await this.setStatus(adjacencyList.length - 1, multi);
|
|
140
|
-
return await multi.exec();
|
|
77
|
+
return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi));
|
|
141
78
|
}
|
|
142
79
|
}
|
|
143
80
|
exports.Await = Await;
|
|
@@ -42,10 +42,10 @@ class Cycle extends activity_1.Activity {
|
|
|
42
42
|
}
|
|
43
43
|
catch (error) {
|
|
44
44
|
if (error instanceof errors_1.GetStateError) {
|
|
45
|
-
this.logger.error('cycle-get-state-error', error);
|
|
45
|
+
this.logger.error('cycle-get-state-error', { error });
|
|
46
46
|
}
|
|
47
47
|
else {
|
|
48
|
-
this.logger.error('cycle-process-error', error);
|
|
48
|
+
this.logger.error('cycle-process-error', { error });
|
|
49
49
|
}
|
|
50
50
|
telemetry.setActivityError(error.message);
|
|
51
51
|
throw error;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Activity } from './activity';
|
|
2
2
|
import { Await } from './await';
|
|
3
3
|
import { Cycle } from './cycle';
|
|
4
|
-
import { Emit } from './emit';
|
|
5
4
|
import { Iterate } from './iterate';
|
|
5
|
+
import { Signal } from './signal';
|
|
6
6
|
import { Trigger } from './trigger';
|
|
7
7
|
import { Worker } from './worker';
|
|
8
8
|
declare const _default: {
|
|
@@ -10,7 +10,7 @@ declare const _default: {
|
|
|
10
10
|
await: typeof Await;
|
|
11
11
|
cycle: typeof Cycle;
|
|
12
12
|
iterate: typeof Iterate;
|
|
13
|
-
|
|
13
|
+
signal: typeof Signal;
|
|
14
14
|
trigger: typeof Trigger;
|
|
15
15
|
worker: typeof Worker;
|
|
16
16
|
};
|
|
@@ -3,8 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const activity_1 = require("./activity");
|
|
4
4
|
const await_1 = require("./await");
|
|
5
5
|
const cycle_1 = require("./cycle");
|
|
6
|
-
const emit_1 = require("./emit");
|
|
7
6
|
const iterate_1 = require("./iterate");
|
|
7
|
+
const signal_1 = require("./signal");
|
|
8
8
|
const trigger_1 = require("./trigger");
|
|
9
9
|
const worker_1 = require("./worker");
|
|
10
10
|
exports.default = {
|
|
@@ -12,7 +12,7 @@ exports.default = {
|
|
|
12
12
|
await: await_1.Await,
|
|
13
13
|
cycle: cycle_1.Cycle,
|
|
14
14
|
iterate: iterate_1.Iterate,
|
|
15
|
-
|
|
15
|
+
signal: signal_1.Signal,
|
|
16
16
|
trigger: trigger_1.Trigger,
|
|
17
17
|
worker: worker_1.Worker,
|
|
18
18
|
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Activity, ActivityType } from './activity';
|
|
2
|
+
import { EngineService } from '../engine';
|
|
3
|
+
import { ActivityData, ActivityMetadata, SignalActivity } from '../../types/activity';
|
|
4
|
+
import { JobState } from '../../types/job';
|
|
5
|
+
declare class Signal extends Activity {
|
|
6
|
+
config: SignalActivity;
|
|
7
|
+
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
8
|
+
process(): Promise<string>;
|
|
9
|
+
mapSignalData(): Record<string, any>;
|
|
10
|
+
mapResolverData(): Record<string, any>;
|
|
11
|
+
/**
|
|
12
|
+
* The signal activity will hook all paused jobs that share the same job key.
|
|
13
|
+
*/
|
|
14
|
+
hookAll(): Promise<string[]>;
|
|
15
|
+
}
|
|
16
|
+
export { Signal };
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Signal = void 0;
|
|
4
|
+
const errors_1 = require("../../modules/errors");
|
|
5
|
+
const activity_1 = require("./activity");
|
|
6
|
+
const collator_1 = require("../collator");
|
|
7
|
+
const mapper_1 = require("../mapper");
|
|
8
|
+
const pipe_1 = require("../pipe");
|
|
9
|
+
const telemetry_1 = require("../telemetry");
|
|
10
|
+
class Signal extends activity_1.Activity {
|
|
11
|
+
constructor(config, data, metadata, hook, engine, context) {
|
|
12
|
+
super(config, data, metadata, hook, engine, context);
|
|
13
|
+
}
|
|
14
|
+
//******** LEG 1 ENTRY ********//
|
|
15
|
+
async process() {
|
|
16
|
+
this.logger.debug('signal-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
17
|
+
let telemetry;
|
|
18
|
+
try {
|
|
19
|
+
//verify entry is allowed
|
|
20
|
+
this.setLeg(1);
|
|
21
|
+
await collator_1.CollatorService.notarizeEntry(this);
|
|
22
|
+
await this.getState();
|
|
23
|
+
telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
|
|
24
|
+
telemetry.startActivitySpan(this.leg);
|
|
25
|
+
//save state and notarize early completion (signals only run leg1)
|
|
26
|
+
const multi = this.store.getMulti();
|
|
27
|
+
this.adjacencyList = await this.filterAdjacent();
|
|
28
|
+
this.mapOutputData();
|
|
29
|
+
this.mapJobData();
|
|
30
|
+
await this.setState(multi);
|
|
31
|
+
await collator_1.CollatorService.notarizeEarlyCompletion(this, multi);
|
|
32
|
+
await this.setStatus(this.adjacencyList.length - 1, multi);
|
|
33
|
+
const multiResponse = await multi.exec();
|
|
34
|
+
//signal to awaken all paused jobs that share the targeted job key
|
|
35
|
+
await this.hookAll();
|
|
36
|
+
//transition to adjacent activities
|
|
37
|
+
const jobStatus = this.resolveStatus(multiResponse);
|
|
38
|
+
const attrs = { 'app.job.jss': jobStatus };
|
|
39
|
+
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
40
|
+
if (messageIds.length) {
|
|
41
|
+
attrs['app.activity.mids'] = messageIds.join(',');
|
|
42
|
+
}
|
|
43
|
+
telemetry.mapActivityAttributes();
|
|
44
|
+
telemetry.setActivityAttributes(attrs);
|
|
45
|
+
return this.context.metadata.aid;
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
if (error instanceof errors_1.GetStateError) {
|
|
49
|
+
this.logger.error('signal-get-state-error', { error });
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
this.logger.error('signal-process-error', { error });
|
|
53
|
+
}
|
|
54
|
+
telemetry.setActivityError(error.message);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
telemetry.endActivitySpan();
|
|
59
|
+
this.logger.debug('signal-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
mapSignalData() {
|
|
63
|
+
if (this.config.signal?.maps) {
|
|
64
|
+
const mapper = new mapper_1.MapperService(this.config.signal.maps, this.context);
|
|
65
|
+
return mapper.mapRules();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
mapResolverData() {
|
|
69
|
+
if (this.config.resolver?.maps) {
|
|
70
|
+
const mapper = new mapper_1.MapperService(this.config.resolver.maps, this.context);
|
|
71
|
+
return mapper.mapRules();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The signal activity will hook all paused jobs that share the same job key.
|
|
76
|
+
*/
|
|
77
|
+
async hookAll() {
|
|
78
|
+
//prep 1) generate `input signal data` (essentially the webhook payload)
|
|
79
|
+
const signalInputData = this.mapSignalData();
|
|
80
|
+
//prep 2) generate data that resolves the job key (per the YAML config)
|
|
81
|
+
const keyResolverData = this.mapResolverData();
|
|
82
|
+
if (this.config.scrub) {
|
|
83
|
+
//self-clean the indexes upon use if configured
|
|
84
|
+
keyResolverData.scrub = true;
|
|
85
|
+
}
|
|
86
|
+
//prep 3) jobKeys can contain multiple indexes (per the YAML config)
|
|
87
|
+
const key_name = pipe_1.Pipe.resolve(this.config.key_name, this.context);
|
|
88
|
+
const key_value = pipe_1.Pipe.resolve(this.config.key_value, this.context);
|
|
89
|
+
const indexQueryFacets = [`${key_name}:${key_value}`];
|
|
90
|
+
//execute: `hookAll` will now resume all paused jobs that share the same job key
|
|
91
|
+
return await this.engine.hookAll(this.config.topic, signalInputData, keyResolverData, indexQueryFacets);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
exports.Signal = Signal;
|
|
@@ -44,10 +44,10 @@ class Trigger extends activity_1.Activity {
|
|
|
44
44
|
}
|
|
45
45
|
catch (error) {
|
|
46
46
|
if (error instanceof errors_1.DuplicateJobError) {
|
|
47
|
-
this.logger.error('duplicate-job-error', error);
|
|
47
|
+
this.logger.error('duplicate-job-error', { error });
|
|
48
48
|
}
|
|
49
49
|
else {
|
|
50
|
-
this.logger.error('trigger-process-error', error);
|
|
50
|
+
this.logger.error('trigger-process-error', { error });
|
|
51
51
|
}
|
|
52
52
|
telemetry.setActivityError(error.message);
|
|
53
53
|
throw error;
|
|
@@ -121,6 +121,7 @@ class Trigger extends activity_1.Activity {
|
|
|
121
121
|
},
|
|
122
122
|
};
|
|
123
123
|
this.context['$self'] = this.context[this.metadata.aid];
|
|
124
|
+
this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
|
|
124
125
|
}
|
|
125
126
|
bindJobMetadataPaths() {
|
|
126
127
|
return serializer_1.MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
|
|
@@ -129,7 +130,7 @@ class Trigger extends activity_1.Activity {
|
|
|
129
130
|
return serializer_1.MDATA_SYMBOLS.ACTIVITY.KEYS.map((key) => `output/metadata/${key}`);
|
|
130
131
|
}
|
|
131
132
|
resolveGranularity() {
|
|
132
|
-
return reporter_1.ReporterService.DEFAULT_GRANULARITY;
|
|
133
|
+
return this.config.stats?.granularity || reporter_1.ReporterService.DEFAULT_GRANULARITY;
|
|
133
134
|
}
|
|
134
135
|
getJobStatus() {
|
|
135
136
|
return this.context.metadata.js;
|
|
@@ -2,10 +2,11 @@ import { Activity } from './activity';
|
|
|
2
2
|
import { EngineService } from '../engine';
|
|
3
3
|
import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
|
|
4
4
|
import { JobState } from '../../types/job';
|
|
5
|
+
import { RedisMulti } from '../../types/redis';
|
|
5
6
|
declare class Worker extends Activity {
|
|
6
7
|
config: WorkerActivity;
|
|
7
8
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
8
9
|
process(): Promise<string>;
|
|
9
|
-
execActivity(): Promise<string>;
|
|
10
|
+
execActivity(multi: RedisMulti): Promise<string>;
|
|
10
11
|
}
|
|
11
12
|
export { Worker };
|