@hotmeshio/hotmesh 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +2 -1
- package/README.md +158 -38
- package/build/package.json +62 -67
- package/build/services/activities/activity.d.ts +58 -7
- package/build/services/activities/activity.js +66 -37
- package/build/services/activities/await.d.ts +101 -0
- package/build/services/activities/await.js +101 -0
- package/build/services/activities/cycle.d.ts +82 -0
- package/build/services/activities/cycle.js +86 -8
- package/build/services/activities/hook.d.ts +139 -1
- package/build/services/activities/hook.js +140 -2
- package/build/services/activities/interrupt.d.ts +112 -0
- package/build/services/activities/interrupt.js +118 -5
- package/build/services/activities/signal.d.ts +108 -3
- package/build/services/activities/signal.js +113 -8
- package/build/services/activities/trigger.d.ts +56 -4
- package/build/services/activities/trigger.js +119 -35
- package/build/services/activities/worker.d.ts +107 -0
- package/build/services/activities/worker.js +107 -0
- package/build/services/collator/index.d.ts +3 -15
- package/build/services/collator/index.js +7 -34
- package/build/services/engine/index.d.ts +18 -2
- package/build/services/engine/index.js +14 -4
- package/build/services/exporter/index.d.ts +2 -0
- package/build/services/exporter/index.js +1 -0
- package/build/services/hotmesh/index.d.ts +471 -236
- package/build/services/hotmesh/index.js +473 -238
- package/build/services/memflow/client.js +2 -2
- package/build/services/memflow/handle.js +1 -1
- package/build/services/memflow/index.d.ts +1 -1
- package/build/services/memflow/index.js +1 -1
- package/build/services/memflow/workflow/all.d.ts +28 -3
- package/build/services/memflow/workflow/all.js +28 -3
- package/build/services/memflow/workflow/context.d.ts +44 -1
- package/build/services/memflow/workflow/context.js +44 -1
- package/build/services/memflow/workflow/didRun.d.ts +23 -3
- package/build/services/memflow/workflow/didRun.js +23 -3
- package/build/services/memflow/workflow/emit.d.ts +43 -4
- package/build/services/memflow/workflow/emit.js +43 -4
- package/build/services/memflow/workflow/enrich.d.ts +32 -4
- package/build/services/memflow/workflow/enrich.js +32 -4
- package/build/services/memflow/workflow/entityMethods.d.ts +54 -7
- package/build/services/memflow/workflow/entityMethods.js +54 -7
- package/build/services/memflow/workflow/execChild.d.ts +96 -8
- package/build/services/memflow/workflow/execChild.js +96 -8
- package/build/services/memflow/workflow/execHook.d.ts +54 -39
- package/build/services/memflow/workflow/execHook.js +52 -38
- package/build/services/memflow/workflow/execHookBatch.d.ts +82 -29
- package/build/services/memflow/workflow/execHookBatch.js +80 -28
- package/build/services/memflow/workflow/hook.d.ts +68 -3
- package/build/services/memflow/workflow/hook.js +69 -4
- package/build/services/memflow/workflow/index.d.ts +65 -10
- package/build/services/memflow/workflow/index.js +65 -10
- package/build/services/memflow/workflow/interrupt.d.ts +50 -4
- package/build/services/memflow/workflow/interrupt.js +50 -4
- package/build/services/memflow/workflow/interruption.d.ts +49 -16
- package/build/services/memflow/workflow/interruption.js +49 -16
- package/build/services/memflow/workflow/isSideEffectAllowed.d.ts +21 -4
- package/build/services/memflow/workflow/isSideEffectAllowed.js +21 -4
- package/build/services/memflow/workflow/proxyActivities.d.ts +70 -42
- package/build/services/memflow/workflow/proxyActivities.js +70 -42
- package/build/services/memflow/workflow/random.d.ts +33 -3
- package/build/services/memflow/workflow/random.js +33 -3
- package/build/services/memflow/workflow/searchMethods.d.ts +49 -2
- package/build/services/memflow/workflow/searchMethods.js +49 -2
- package/build/services/memflow/workflow/signal.d.ts +51 -22
- package/build/services/memflow/workflow/signal.js +52 -23
- package/build/services/memflow/workflow/sleepFor.d.ts +57 -18
- package/build/services/memflow/workflow/sleepFor.js +57 -18
- package/build/services/memflow/workflow/trace.d.ts +39 -6
- package/build/services/memflow/workflow/trace.js +39 -6
- package/build/services/memflow/workflow/waitFor.d.ts +55 -18
- package/build/services/memflow/workflow/waitFor.js +55 -18
- package/build/services/store/index.d.ts +1 -1
- package/build/services/store/providers/postgres/postgres.d.ts +1 -1
- package/build/services/store/providers/postgres/postgres.js +4 -3
- package/build/services/telemetry/index.js +6 -0
- package/build/types/activity.d.ts +1 -1
- package/build/types/hotmesh.d.ts +1 -1
- package/build/types/job.d.ts +1 -1
- package/build/types/memflow.d.ts +1 -1
- package/build/types/quorum.d.ts +2 -2
- package/build/vitest.config.d.ts +2 -0
- package/build/vitest.config.js +18 -0
- package/package.json +62 -67
- package/vitest.config.ts +17 -0
|
@@ -1,14 +1,67 @@
|
|
|
1
1
|
import { EngineService } from '../engine';
|
|
2
2
|
import { ActivityData, ActivityMetadata, ActivityType, TriggerActivity } from '../../types/activity';
|
|
3
|
-
import { JobState, ExtensionType
|
|
3
|
+
import { JobState, ExtensionType } from '../../types/job';
|
|
4
4
|
import { ProviderTransaction } from '../../types/provider';
|
|
5
|
-
import { StringScalarType } from '../../types/serializer';
|
|
6
5
|
import { Activity } from './activity';
|
|
6
|
+
/**
|
|
7
|
+
* The entry point for every workflow graph. Each graph must have exactly
|
|
8
|
+
* one `trigger` activity, which executes when the graph's `subscribes`
|
|
9
|
+
* topic receives a message (via `hotMesh.pub` or `hotMesh.pubsub`).
|
|
10
|
+
*
|
|
11
|
+
* The trigger initializes the job, sets its unique ID and key, binds
|
|
12
|
+
* the incoming payload as the trigger's output data, and transitions
|
|
13
|
+
* to adjacent activities defined in the `transitions` section.
|
|
14
|
+
*
|
|
15
|
+
* ## YAML Configuration
|
|
16
|
+
*
|
|
17
|
+
* ```yaml
|
|
18
|
+
* app:
|
|
19
|
+
* id: myapp
|
|
20
|
+
* version: '1'
|
|
21
|
+
* graphs:
|
|
22
|
+
* - subscribes: order.placed # the trigger fires when this topic receives a message
|
|
23
|
+
* publishes: order.processed # emitted when the graph completes
|
|
24
|
+
* expire: 120
|
|
25
|
+
*
|
|
26
|
+
* activities:
|
|
27
|
+
* t1:
|
|
28
|
+
* type: trigger
|
|
29
|
+
* entity: '{$self.input.data.entityType}'
|
|
30
|
+
* job:
|
|
31
|
+
* maps:
|
|
32
|
+
* myField: '{$self.output.data.inputField}'
|
|
33
|
+
* stats:
|
|
34
|
+
* id: '{$self.input.data.workflowId}'
|
|
35
|
+
* key: '{$self.input.data.parentId}'
|
|
36
|
+
* parent: '{$self.input.data.originJobId}'
|
|
37
|
+
* adjacent: '{$self.input.data.parentJobId}'
|
|
38
|
+
*
|
|
39
|
+
* process:
|
|
40
|
+
* type: worker
|
|
41
|
+
* topic: order.process
|
|
42
|
+
*
|
|
43
|
+
* transitions:
|
|
44
|
+
* t1:
|
|
45
|
+
* - to: process
|
|
46
|
+
* ```
|
|
47
|
+
*
|
|
48
|
+
* ## Key Behaviors
|
|
49
|
+
*
|
|
50
|
+
* - **Job ID Resolution**: If `stats.id` is provided, it resolves via `@pipe`
|
|
51
|
+
* expressions against the input data. Otherwise a UUID is generated.
|
|
52
|
+
* - **Duplicate Detection**: If a job with the same ID already exists,
|
|
53
|
+
* a `DuplicateJobError` is thrown (unless it's a crash-recovery scenario).
|
|
54
|
+
* - **Pending Mode**: When invoked with `{ pending: <seconds> }`, the trigger
|
|
55
|
+
* creates the job but does not transition to children until resumed.
|
|
56
|
+
* - **Crash Recovery**: Uses a 3-step inception protocol with GUID ledger
|
|
57
|
+
* to ensure atomic job creation survives process crashes.
|
|
58
|
+
*
|
|
59
|
+
* @see {@link TriggerActivity} for the TypeScript interface
|
|
60
|
+
*/
|
|
7
61
|
declare class Trigger extends Activity {
|
|
8
62
|
config: TriggerActivity;
|
|
9
63
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
10
64
|
process(options?: ExtensionType): Promise<string>;
|
|
11
|
-
transitionAndLogAdjacent(options: ExtensionType, jobStatus: JobStatus, attrs: StringScalarType): Promise<void>;
|
|
12
65
|
/**
|
|
13
66
|
* `pending` flows will not transition from the trigger to adjacent children until resumed
|
|
14
67
|
*/
|
|
@@ -31,7 +84,6 @@ declare class Trigger extends Activity {
|
|
|
31
84
|
getJobStatus(): number;
|
|
32
85
|
resolveJobId(context: Partial<JobState>): string;
|
|
33
86
|
resolveJobKey(context: Partial<JobState>): string;
|
|
34
|
-
setStateNX(status?: number, entity?: string | undefined): Promise<void>;
|
|
35
87
|
setStats(transaction?: ProviderTransaction): Promise<void>;
|
|
36
88
|
}
|
|
37
89
|
export { Trigger };
|
|
@@ -10,6 +10,61 @@ const serializer_1 = require("../serializer");
|
|
|
10
10
|
const telemetry_1 = require("../telemetry");
|
|
11
11
|
const mapper_1 = require("../mapper");
|
|
12
12
|
const activity_1 = require("./activity");
|
|
13
|
+
/**
|
|
14
|
+
* The entry point for every workflow graph. Each graph must have exactly
|
|
15
|
+
* one `trigger` activity, which executes when the graph's `subscribes`
|
|
16
|
+
* topic receives a message (via `hotMesh.pub` or `hotMesh.pubsub`).
|
|
17
|
+
*
|
|
18
|
+
* The trigger initializes the job, sets its unique ID and key, binds
|
|
19
|
+
* the incoming payload as the trigger's output data, and transitions
|
|
20
|
+
* to adjacent activities defined in the `transitions` section.
|
|
21
|
+
*
|
|
22
|
+
* ## YAML Configuration
|
|
23
|
+
*
|
|
24
|
+
* ```yaml
|
|
25
|
+
* app:
|
|
26
|
+
* id: myapp
|
|
27
|
+
* version: '1'
|
|
28
|
+
* graphs:
|
|
29
|
+
* - subscribes: order.placed # the trigger fires when this topic receives a message
|
|
30
|
+
* publishes: order.processed # emitted when the graph completes
|
|
31
|
+
* expire: 120
|
|
32
|
+
*
|
|
33
|
+
* activities:
|
|
34
|
+
* t1:
|
|
35
|
+
* type: trigger
|
|
36
|
+
* entity: '{$self.input.data.entityType}'
|
|
37
|
+
* job:
|
|
38
|
+
* maps:
|
|
39
|
+
* myField: '{$self.output.data.inputField}'
|
|
40
|
+
* stats:
|
|
41
|
+
* id: '{$self.input.data.workflowId}'
|
|
42
|
+
* key: '{$self.input.data.parentId}'
|
|
43
|
+
* parent: '{$self.input.data.originJobId}'
|
|
44
|
+
* adjacent: '{$self.input.data.parentJobId}'
|
|
45
|
+
*
|
|
46
|
+
* process:
|
|
47
|
+
* type: worker
|
|
48
|
+
* topic: order.process
|
|
49
|
+
*
|
|
50
|
+
* transitions:
|
|
51
|
+
* t1:
|
|
52
|
+
* - to: process
|
|
53
|
+
* ```
|
|
54
|
+
*
|
|
55
|
+
* ## Key Behaviors
|
|
56
|
+
*
|
|
57
|
+
* - **Job ID Resolution**: If `stats.id` is provided, it resolves via `@pipe`
|
|
58
|
+
* expressions against the input data. Otherwise a UUID is generated.
|
|
59
|
+
* - **Duplicate Detection**: If a job with the same ID already exists,
|
|
60
|
+
* a `DuplicateJobError` is thrown (unless it's a crash-recovery scenario).
|
|
61
|
+
* - **Pending Mode**: When invoked with `{ pending: <seconds> }`, the trigger
|
|
62
|
+
* creates the job but does not transition to children until resumed.
|
|
63
|
+
* - **Crash Recovery**: Uses a 3-step inception protocol with GUID ledger
|
|
64
|
+
* to ensure atomic job creation survives process crashes.
|
|
65
|
+
*
|
|
66
|
+
* @see {@link TriggerActivity} for the TypeScript interface
|
|
67
|
+
*/
|
|
13
68
|
class Trigger extends activity_1.Activity {
|
|
14
69
|
constructor(config, data, metadata, hook, engine, context) {
|
|
15
70
|
super(config, data, metadata, hook, engine, context);
|
|
@@ -30,40 +85,84 @@ class Trigger extends activity_1.Activity {
|
|
|
30
85
|
const initialStatus = this.initStatus(options, this.adjacencyList.length);
|
|
31
86
|
//config.entity is a pipe expression; if 'entity' exists, it will resolve
|
|
32
87
|
const resolvedEntity = new mapper_1.MapperService({ entity: this.config.entity }, this.context).mapRules()?.entity;
|
|
33
|
-
|
|
88
|
+
const msgGuid = this.context.metadata.guid || (0, utils_1.guid)();
|
|
89
|
+
this.context.metadata.guid = msgGuid;
|
|
90
|
+
const { id: appId } = await this.engine.getVID();
|
|
91
|
+
//═══ Step 1: Inception (atomic job creation + GUID seed) ═══
|
|
92
|
+
const txn1 = this.store.transact();
|
|
93
|
+
await this.store.setStateNX(this.context.metadata.jid, appId, initialStatus, options?.entity || resolvedEntity, txn1);
|
|
94
|
+
await this.store.collateSynthetic(this.context.metadata.jid, msgGuid, collator_1.CollatorService.WEIGHTS.STEP1_WORK, txn1);
|
|
95
|
+
const results1 = (await txn1.exec());
|
|
96
|
+
const jobCreated = Number(results1[0]) > 0;
|
|
97
|
+
const guidValue = Number(results1[1]);
|
|
98
|
+
if (!jobCreated) {
|
|
99
|
+
if (guidValue > collator_1.CollatorService.WEIGHTS.STEP1_WORK) {
|
|
100
|
+
//crash recovery: GUID was seeded on a prior attempt; resume
|
|
101
|
+
this.guidLedger = guidValue;
|
|
102
|
+
this.logger.info('trigger-crash-recovery', {
|
|
103
|
+
job_id: this.context.metadata.jid,
|
|
104
|
+
guid: msgGuid,
|
|
105
|
+
guidLedger: guidValue,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
//true duplicate: another process owns this job
|
|
110
|
+
throw new errors_1.DuplicateJobError(this.context.metadata.jid);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
34
113
|
await this.setStatus(initialStatus);
|
|
35
114
|
this.bindSearchData(options);
|
|
36
115
|
this.bindMarkerData(options);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
await this.
|
|
116
|
+
//═══ Step 2: Work (state + children + GUID Step 2) ═══
|
|
117
|
+
if (!collator_1.CollatorService.isGuidStep2Done(this.guidLedger)) {
|
|
118
|
+
const txn2 = this.store.transact();
|
|
119
|
+
await this.setState(txn2);
|
|
120
|
+
await this.setStats(txn2);
|
|
121
|
+
if (options?.pending) {
|
|
122
|
+
await this.setExpired(options.pending, txn2);
|
|
123
|
+
}
|
|
124
|
+
//publish children (unless pending or job already complete)
|
|
125
|
+
if (isNaN(options?.pending) && this.adjacencyList.length && initialStatus > 0) {
|
|
126
|
+
for (const child of this.adjacencyList) {
|
|
127
|
+
await this.engine.router?.publishMessage(null, child, txn2);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
await collator_1.CollatorService.notarizeStep2(this, msgGuid, txn2);
|
|
131
|
+
await txn2.exec();
|
|
42
132
|
}
|
|
43
|
-
|
|
44
|
-
await transaction.exec();
|
|
133
|
+
//best-effort parent notification
|
|
45
134
|
this.execAdjacentParent();
|
|
46
|
-
|
|
135
|
+
//═══ Step 3: Completion (if job immediately complete) ═══
|
|
136
|
+
//NOTE: runJobCompletionTasks is non-transactional here because
|
|
137
|
+
//the trigger runs inline within pub(). Subscribers register AFTER
|
|
138
|
+
//pub() returns, so pub notifications must be fire-and-forget to
|
|
139
|
+
//avoid a race. The GUID marker commits in its own transaction.
|
|
47
140
|
const jobStatus = Number(this.context.metadata.js);
|
|
141
|
+
const needsCompletion = this.shouldEmit() ||
|
|
142
|
+
this.isJobComplete(jobStatus) ||
|
|
143
|
+
this.shouldPersistJob();
|
|
144
|
+
if (needsCompletion && !collator_1.CollatorService.isGuidStep3Done(this.guidLedger)) {
|
|
145
|
+
await this.engine.runJobCompletionTasks(this.context, { emit: !this.isJobComplete(jobStatus) && !this.shouldPersistJob() });
|
|
146
|
+
//NOTE: notarizeStep3 is fire-and-forget for the trigger.
|
|
147
|
+
//pubOneTimeSubs (inside runJobCompletionTasks) sends a NOTIFY
|
|
148
|
+
//that must not be processed until registerJobCallback runs
|
|
149
|
+
//AFTER pub() returns. An `await` here yields to the event loop,
|
|
150
|
+
//which could deliver the NOTIFY before the callback is registered.
|
|
151
|
+
const txn3 = this.store.transact();
|
|
152
|
+
collator_1.CollatorService.notarizeStep3(this, msgGuid, txn3)
|
|
153
|
+
.then(() => txn3.exec())
|
|
154
|
+
.catch((err) => this.logger.error('trigger-notarize-step3-error', { error: err }));
|
|
155
|
+
}
|
|
156
|
+
//telemetry
|
|
157
|
+
telemetry.mapActivityAttributes();
|
|
48
158
|
telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
|
|
49
159
|
const attrs = { 'app.job.jss': jobStatus };
|
|
50
|
-
await this.transitionAndLogAdjacent(options, jobStatus, attrs);
|
|
51
160
|
telemetry.setActivityAttributes(attrs);
|
|
52
161
|
return this.context.metadata.jid;
|
|
53
162
|
}
|
|
54
163
|
catch (error) {
|
|
55
164
|
telemetry?.setActivityError(error.message);
|
|
56
165
|
if (error instanceof errors_1.DuplicateJobError) {
|
|
57
|
-
//todo: verify baseline in x-AZ rollover
|
|
58
|
-
await (0, utils_1.sleepFor)(1000);
|
|
59
|
-
const isOverage = await collator_1.CollatorService.isInceptionOverage(this, this.context.metadata.guid);
|
|
60
|
-
if (isOverage) {
|
|
61
|
-
this.logger.info('trigger-collation-overage', {
|
|
62
|
-
job_id: error.jobId,
|
|
63
|
-
guid: this.context.metadata.guid,
|
|
64
|
-
});
|
|
65
|
-
return;
|
|
66
|
-
}
|
|
67
166
|
this.logger.error('duplicate-job-error', {
|
|
68
167
|
job_id: error.jobId,
|
|
69
168
|
guid: this.context.metadata.guid,
|
|
@@ -84,15 +183,6 @@ class Trigger extends activity_1.Activity {
|
|
|
84
183
|
});
|
|
85
184
|
}
|
|
86
185
|
}
|
|
87
|
-
async transitionAndLogAdjacent(options = {}, jobStatus, attrs) {
|
|
88
|
-
//todo: enable resume from pending state
|
|
89
|
-
if (isNaN(options.pending)) {
|
|
90
|
-
const messageIds = await this.transition(this.adjacencyList, jobStatus);
|
|
91
|
-
if (messageIds.length) {
|
|
92
|
-
attrs['app.activity.mids'] = messageIds.join(',');
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
186
|
/**
|
|
97
187
|
* `pending` flows will not transition from the trigger to adjacent children until resumed
|
|
98
188
|
*/
|
|
@@ -231,12 +321,6 @@ class Trigger extends activity_1.Activity {
|
|
|
231
321
|
const jobKey = this.config.stats?.key;
|
|
232
322
|
return jobKey ? pipe_1.Pipe.resolve(jobKey, context) : '';
|
|
233
323
|
}
|
|
234
|
-
async setStateNX(status, entity) {
|
|
235
|
-
const jobId = this.context.metadata.jid;
|
|
236
|
-
if (!await this.store.setStateNX(jobId, this.engine.appId, status, entity)) {
|
|
237
|
-
throw new errors_1.DuplicateJobError(jobId);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
324
|
async setStats(transaction) {
|
|
241
325
|
const md = this.context.metadata;
|
|
242
326
|
if (md.key && this.config.stats?.measures) {
|
|
@@ -3,6 +3,113 @@ import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '..
|
|
|
3
3
|
import { JobState } from '../../types/job';
|
|
4
4
|
import { ProviderTransaction } from '../../types/provider';
|
|
5
5
|
import { Activity } from './activity';
|
|
6
|
+
/**
|
|
7
|
+
* Dispatches work to a registered callback function. The `worker` activity
|
|
8
|
+
* publishes a message to its configured `topic` stream, where a worker
|
|
9
|
+
* process picks it up, executes the callback, and returns a response
|
|
10
|
+
* that the engine captures as the activity's output.
|
|
11
|
+
*
|
|
12
|
+
* ## YAML Configuration
|
|
13
|
+
*
|
|
14
|
+
* ```yaml
|
|
15
|
+
* app:
|
|
16
|
+
* id: myapp
|
|
17
|
+
* version: '1'
|
|
18
|
+
* graphs:
|
|
19
|
+
* - subscribes: order.placed
|
|
20
|
+
* expire: 120
|
|
21
|
+
*
|
|
22
|
+
* activities:
|
|
23
|
+
* t1:
|
|
24
|
+
* type: trigger
|
|
25
|
+
*
|
|
26
|
+
* a1:
|
|
27
|
+
* type: worker
|
|
28
|
+
* topic: work.do # matches the registered worker topic
|
|
29
|
+
* input:
|
|
30
|
+
* schema:
|
|
31
|
+
* type: object
|
|
32
|
+
* properties:
|
|
33
|
+
* x: { type: string }
|
|
34
|
+
* maps:
|
|
35
|
+
* x: '{t1.output.data.inputField}'
|
|
36
|
+
* output:
|
|
37
|
+
* schema:
|
|
38
|
+
* type: object
|
|
39
|
+
* properties:
|
|
40
|
+
* y: { type: string }
|
|
41
|
+
* job:
|
|
42
|
+
* maps:
|
|
43
|
+
* result: '{$self.output.data.y}'
|
|
44
|
+
*
|
|
45
|
+
* transitions:
|
|
46
|
+
* t1:
|
|
47
|
+
* - to: a1
|
|
48
|
+
* ```
|
|
49
|
+
*
|
|
50
|
+
* ## Worker Registration (JavaScript)
|
|
51
|
+
*
|
|
52
|
+
* Workers are registered at initialization time via the `workers` array
|
|
53
|
+
* in `HotMesh.init`. Each worker binds a `topic` to a `callback` function.
|
|
54
|
+
*
|
|
55
|
+
* ```typescript
|
|
56
|
+
* const hotMesh = await HotMesh.init({
|
|
57
|
+
* appId: 'myapp',
|
|
58
|
+
* engine: { connection },
|
|
59
|
+
* workers: [{
|
|
60
|
+
* topic: 'work.do',
|
|
61
|
+
* connection,
|
|
62
|
+
* callback: async (data: StreamData) => ({
|
|
63
|
+
* metadata: { ...data.metadata },
|
|
64
|
+
* data: { y: `${data.data.x} transformed` }
|
|
65
|
+
* })
|
|
66
|
+
* }]
|
|
67
|
+
* });
|
|
68
|
+
* ```
|
|
69
|
+
*
|
|
70
|
+
* ## Retry Policy
|
|
71
|
+
*
|
|
72
|
+
* Retry behavior is configured at the **worker level** (not in YAML) via
|
|
73
|
+
* the `retryPolicy` option. Failed callbacks are retried with exponential
|
|
74
|
+
* backoff until `maximumAttempts` is exhausted. The `maximumInterval` caps
|
|
75
|
+
* the delay between retries.
|
|
76
|
+
*
|
|
77
|
+
* ```typescript
|
|
78
|
+
* const hotMesh = await HotMesh.init({
|
|
79
|
+
* appId: 'myapp',
|
|
80
|
+
* engine: { connection },
|
|
81
|
+
* workers: [{
|
|
82
|
+
* topic: 'work.backoff',
|
|
83
|
+
* connection,
|
|
84
|
+
* retryPolicy: {
|
|
85
|
+
* maximumAttempts: 5, // retry up to 5 times
|
|
86
|
+
* backoffCoefficient: 2, // exponential: 2^0, 2^1, 2^2, ... seconds
|
|
87
|
+
* maximumInterval: '30s', // cap delay at 30 seconds
|
|
88
|
+
* },
|
|
89
|
+
* callback: async (data: StreamData) => {
|
|
90
|
+
* const result = await doWork(data.data);
|
|
91
|
+
* return {
|
|
92
|
+
* code: 200,
|
|
93
|
+
* status: StreamStatus.SUCCESS,
|
|
94
|
+
* metadata: { ...data.metadata },
|
|
95
|
+
* data: { result },
|
|
96
|
+
* } as StreamDataResponse;
|
|
97
|
+
* },
|
|
98
|
+
* }]
|
|
99
|
+
* });
|
|
100
|
+
* ```
|
|
101
|
+
*
|
|
102
|
+
* ## Execution Model
|
|
103
|
+
*
|
|
104
|
+
* Worker is a **Category A (duplex)** activity:
|
|
105
|
+
* - **Leg 1** (`process`): Maps input data and publishes a message to the
|
|
106
|
+
* worker's topic stream.
|
|
107
|
+
* - **Leg 2** (`processEvent`, inherited): Receives the worker's response,
|
|
108
|
+
* maps output data, and executes the step protocol to transition to
|
|
109
|
+
* adjacent activities.
|
|
110
|
+
*
|
|
111
|
+
* @see {@link WorkerActivity} for the TypeScript interface
|
|
112
|
+
*/
|
|
6
113
|
declare class Worker extends Activity {
|
|
7
114
|
config: WorkerActivity;
|
|
8
115
|
constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
|
|
@@ -7,6 +7,113 @@ const collator_1 = require("../collator");
|
|
|
7
7
|
const pipe_1 = require("../pipe");
|
|
8
8
|
const telemetry_1 = require("../telemetry");
|
|
9
9
|
const activity_1 = require("./activity");
|
|
10
|
+
/**
|
|
11
|
+
* Dispatches work to a registered callback function. The `worker` activity
|
|
12
|
+
* publishes a message to its configured `topic` stream, where a worker
|
|
13
|
+
* process picks it up, executes the callback, and returns a response
|
|
14
|
+
* that the engine captures as the activity's output.
|
|
15
|
+
*
|
|
16
|
+
* ## YAML Configuration
|
|
17
|
+
*
|
|
18
|
+
* ```yaml
|
|
19
|
+
* app:
|
|
20
|
+
* id: myapp
|
|
21
|
+
* version: '1'
|
|
22
|
+
* graphs:
|
|
23
|
+
* - subscribes: order.placed
|
|
24
|
+
* expire: 120
|
|
25
|
+
*
|
|
26
|
+
* activities:
|
|
27
|
+
* t1:
|
|
28
|
+
* type: trigger
|
|
29
|
+
*
|
|
30
|
+
* a1:
|
|
31
|
+
* type: worker
|
|
32
|
+
* topic: work.do # matches the registered worker topic
|
|
33
|
+
* input:
|
|
34
|
+
* schema:
|
|
35
|
+
* type: object
|
|
36
|
+
* properties:
|
|
37
|
+
* x: { type: string }
|
|
38
|
+
* maps:
|
|
39
|
+
* x: '{t1.output.data.inputField}'
|
|
40
|
+
* output:
|
|
41
|
+
* schema:
|
|
42
|
+
* type: object
|
|
43
|
+
* properties:
|
|
44
|
+
* y: { type: string }
|
|
45
|
+
* job:
|
|
46
|
+
* maps:
|
|
47
|
+
* result: '{$self.output.data.y}'
|
|
48
|
+
*
|
|
49
|
+
* transitions:
|
|
50
|
+
* t1:
|
|
51
|
+
* - to: a1
|
|
52
|
+
* ```
|
|
53
|
+
*
|
|
54
|
+
* ## Worker Registration (JavaScript)
|
|
55
|
+
*
|
|
56
|
+
* Workers are registered at initialization time via the `workers` array
|
|
57
|
+
* in `HotMesh.init`. Each worker binds a `topic` to a `callback` function.
|
|
58
|
+
*
|
|
59
|
+
* ```typescript
|
|
60
|
+
* const hotMesh = await HotMesh.init({
|
|
61
|
+
* appId: 'myapp',
|
|
62
|
+
* engine: { connection },
|
|
63
|
+
* workers: [{
|
|
64
|
+
* topic: 'work.do',
|
|
65
|
+
* connection,
|
|
66
|
+
* callback: async (data: StreamData) => ({
|
|
67
|
+
* metadata: { ...data.metadata },
|
|
68
|
+
* data: { y: `${data.data.x} transformed` }
|
|
69
|
+
* })
|
|
70
|
+
* }]
|
|
71
|
+
* });
|
|
72
|
+
* ```
|
|
73
|
+
*
|
|
74
|
+
* ## Retry Policy
|
|
75
|
+
*
|
|
76
|
+
* Retry behavior is configured at the **worker level** (not in YAML) via
|
|
77
|
+
* the `retryPolicy` option. Failed callbacks are retried with exponential
|
|
78
|
+
* backoff until `maximumAttempts` is exhausted. The `maximumInterval` caps
|
|
79
|
+
* the delay between retries.
|
|
80
|
+
*
|
|
81
|
+
* ```typescript
|
|
82
|
+
* const hotMesh = await HotMesh.init({
|
|
83
|
+
* appId: 'myapp',
|
|
84
|
+
* engine: { connection },
|
|
85
|
+
* workers: [{
|
|
86
|
+
* topic: 'work.backoff',
|
|
87
|
+
* connection,
|
|
88
|
+
* retryPolicy: {
|
|
89
|
+
* maximumAttempts: 5, // retry up to 5 times
|
|
90
|
+
* backoffCoefficient: 2, // exponential: 2^0, 2^1, 2^2, ... seconds
|
|
91
|
+
* maximumInterval: '30s', // cap delay at 30 seconds
|
|
92
|
+
* },
|
|
93
|
+
* callback: async (data: StreamData) => {
|
|
94
|
+
* const result = await doWork(data.data);
|
|
95
|
+
* return {
|
|
96
|
+
* code: 200,
|
|
97
|
+
* status: StreamStatus.SUCCESS,
|
|
98
|
+
* metadata: { ...data.metadata },
|
|
99
|
+
* data: { result },
|
|
100
|
+
* } as StreamDataResponse;
|
|
101
|
+
* },
|
|
102
|
+
* }]
|
|
103
|
+
* });
|
|
104
|
+
* ```
|
|
105
|
+
*
|
|
106
|
+
* ## Execution Model
|
|
107
|
+
*
|
|
108
|
+
* Worker is a **Category A (duplex)** activity:
|
|
109
|
+
* - **Leg 1** (`process`): Maps input data and publishes a message to the
|
|
110
|
+
* worker's topic stream.
|
|
111
|
+
* - **Leg 2** (`processEvent`, inherited): Receives the worker's response,
|
|
112
|
+
* maps output data, and executes the step protocol to transition to
|
|
113
|
+
* adjacent activities.
|
|
114
|
+
*
|
|
115
|
+
* @see {@link WorkerActivity} for the TypeScript interface
|
|
116
|
+
*/
|
|
10
117
|
class Worker extends activity_1.Activity {
|
|
11
118
|
constructor(config, data, metadata, hook, engine, context) {
|
|
12
119
|
super(config, data, metadata, hook, engine, context);
|
|
@@ -91,18 +91,11 @@ declare class CollatorService {
|
|
|
91
91
|
*/
|
|
92
92
|
static notarizeStep3(activity: Activity, guid: string, transaction: ProviderTransaction): Promise<void>;
|
|
93
93
|
/**
|
|
94
|
-
* Finalize: close the activity to new Leg2 GUIDs (+
|
|
94
|
+
* Finalize: close the activity to new Leg2 GUIDs (+200T).
|
|
95
|
+
* Sets pos 1 to 2 (finalized).
|
|
95
96
|
* Only for non-cycle activities after final SUCCESS/ERROR.
|
|
96
97
|
*/
|
|
97
98
|
static notarizeFinalize(activity: Activity, transaction: ProviderTransaction): Promise<void>;
|
|
98
|
-
/**
|
|
99
|
-
* sets the synthetic inception key (in case an overage occurs).
|
|
100
|
-
*/
|
|
101
|
-
static notarizeInception(activity: Activity, guid: string, transaction: ProviderTransaction): Promise<void>;
|
|
102
|
-
/**
|
|
103
|
-
* ignore those ID collisions that are due to re-entry overages
|
|
104
|
-
*/
|
|
105
|
-
static isInceptionOverage(activity: Activity, guid: string): Promise<boolean>;
|
|
106
99
|
/**
|
|
107
100
|
* Check if Step 1 (work done) is complete on the GUID ledger.
|
|
108
101
|
* Position 5 (10B digit) > 0.
|
|
@@ -159,15 +152,10 @@ declare class CollatorService {
|
|
|
159
152
|
* If pos 3 > 1 and pos 4 == 1, it's a stale/replayed message.
|
|
160
153
|
*
|
|
161
154
|
* Leg2 enter: pos 4 (100B) must be > 0 (Leg1 complete, reentry authorized).
|
|
162
|
-
* pos
|
|
155
|
+
* pos 1 (100T) must be < 2 (not finalized) — cycle activities exempt.
|
|
163
156
|
*/
|
|
164
157
|
static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
|
|
165
158
|
static getDimensionsById(ancestors: string[], dad: string): Record<string, string>;
|
|
166
|
-
/**
|
|
167
|
-
* All non-trigger activities are assigned a status seed by their parent.
|
|
168
|
-
* Seed: 100000000000000 (pos 1 = 1, authorized for entry)
|
|
169
|
-
*/
|
|
170
|
-
static getSeed(): string;
|
|
171
159
|
/**
|
|
172
160
|
* All trigger activities are assigned a status seed in a completed state.
|
|
173
161
|
* Seed: 101100000000001 (authorized, 1 Leg1 entry, Leg1 complete, 1 Leg2 entry)
|
|
@@ -131,7 +131,8 @@ class CollatorService {
|
|
|
131
131
|
await activity.store.collateSynthetic(jid, guid, this.WEIGHTS.STEP3_CLEANUP, transaction);
|
|
132
132
|
}
|
|
133
133
|
/**
|
|
134
|
-
* Finalize: close the activity to new Leg2 GUIDs (+
|
|
134
|
+
* Finalize: close the activity to new Leg2 GUIDs (+200T).
|
|
135
|
+
* Sets pos 1 to 2 (finalized).
|
|
135
136
|
* Only for non-cycle activities after final SUCCESS/ERROR.
|
|
136
137
|
*/
|
|
137
138
|
static async notarizeFinalize(activity, transaction) {
|
|
@@ -140,27 +141,6 @@ class CollatorService {
|
|
|
140
141
|
}
|
|
141
142
|
}
|
|
142
143
|
// ──────────────────────────────────────────────────
|
|
143
|
-
// Inception (trigger duplicate detection)
|
|
144
|
-
// ──────────────────────────────────────────────────
|
|
145
|
-
/**
|
|
146
|
-
* sets the synthetic inception key (in case an overage occurs).
|
|
147
|
-
*/
|
|
148
|
-
static async notarizeInception(activity, guid, transaction) {
|
|
149
|
-
if (guid) {
|
|
150
|
-
await activity.store.collateSynthetic(activity.context.metadata.jid, guid, 1000000, transaction);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
/**
|
|
154
|
-
* ignore those ID collisions that are due to re-entry overages
|
|
155
|
-
*/
|
|
156
|
-
static async isInceptionOverage(activity, guid) {
|
|
157
|
-
if (guid) {
|
|
158
|
-
const amount = await activity.store.collateSynthetic(activity.context.metadata.jid, guid, 1000000);
|
|
159
|
-
return amount > 1000000;
|
|
160
|
-
}
|
|
161
|
-
return false;
|
|
162
|
-
}
|
|
163
|
-
// ──────────────────────────────────────────────────
|
|
164
144
|
// GUID ledger extraction (step-level resume)
|
|
165
145
|
// ──────────────────────────────────────────────────
|
|
166
146
|
/**
|
|
@@ -270,7 +250,7 @@ class CollatorService {
|
|
|
270
250
|
* If pos 3 > 1 and pos 4 == 1, it's a stale/replayed message.
|
|
271
251
|
*
|
|
272
252
|
* Leg2 enter: pos 4 (100B) must be > 0 (Leg1 complete, reentry authorized).
|
|
273
|
-
* pos
|
|
253
|
+
* pos 1 (100T) must be < 2 (not finalized) — cycle activities exempt.
|
|
274
254
|
*/
|
|
275
255
|
static verifyInteger(amount, leg, stage) {
|
|
276
256
|
let faultType;
|
|
@@ -288,13 +268,13 @@ class CollatorService {
|
|
|
288
268
|
}
|
|
289
269
|
else if (leg === 2 && stage === 'enter') {
|
|
290
270
|
const leg1Complete = this.getDigitAtPosition(amount, 4);
|
|
291
|
-
const finalized = this.getDigitAtPosition(amount,
|
|
271
|
+
const finalized = this.getDigitAtPosition(amount, 1);
|
|
292
272
|
if (leg1Complete === 0) {
|
|
293
273
|
//Leg1 not complete — reentry not authorized
|
|
294
274
|
faultType = collator_1.CollationFaultType.FORBIDDEN;
|
|
295
275
|
}
|
|
296
|
-
else if (finalized
|
|
297
|
-
//activity finalized — no new Leg2 GUIDs accepted
|
|
276
|
+
else if (finalized >= 2) {
|
|
277
|
+
//activity finalized (pos 1 = 2) — no new Leg2 GUIDs accepted
|
|
298
278
|
faultType = collator_1.CollationFaultType.INACTIVE;
|
|
299
279
|
}
|
|
300
280
|
}
|
|
@@ -322,13 +302,6 @@ class CollatorService {
|
|
|
322
302
|
// ──────────────────────────────────────────────────
|
|
323
303
|
// Seeds
|
|
324
304
|
// ──────────────────────────────────────────────────
|
|
325
|
-
/**
|
|
326
|
-
* All non-trigger activities are assigned a status seed by their parent.
|
|
327
|
-
* Seed: 100000000000000 (pos 1 = 1, authorized for entry)
|
|
328
|
-
*/
|
|
329
|
-
static getSeed() {
|
|
330
|
-
return '100000000000000';
|
|
331
|
-
}
|
|
332
305
|
/**
|
|
333
306
|
* All trigger activities are assigned a status seed in a completed state.
|
|
334
307
|
* Seed: 101100000000001 (authorized, 1 Leg1 entry, Leg1 complete, 1 Leg2 entry)
|
|
@@ -405,7 +378,7 @@ CollatorService.targetLength = 15;
|
|
|
405
378
|
*/
|
|
406
379
|
CollatorService.WEIGHTS = {
|
|
407
380
|
AUTH: 100000000000000,
|
|
408
|
-
FINALIZE:
|
|
381
|
+
FINALIZE: 200000000000000,
|
|
409
382
|
LEG1_ENTRY: 1000000000000,
|
|
410
383
|
LEG1_COMPLETE: 100000000000,
|
|
411
384
|
STEP1_WORK: 10000000000,
|
|
@@ -31,11 +31,17 @@ declare class EngineService {
|
|
|
31
31
|
appId: string;
|
|
32
32
|
guid: string;
|
|
33
33
|
exporter: ExporterService | null;
|
|
34
|
+
/** @hidden */
|
|
34
35
|
search: SearchService<ProviderClient> | null;
|
|
36
|
+
/** @hidden */
|
|
35
37
|
store: StoreService<ProviderClient, ProviderTransaction> | null;
|
|
38
|
+
/** @hidden */
|
|
36
39
|
stream: StreamService<ProviderClient, ProviderTransaction> | null;
|
|
40
|
+
/** @hidden */
|
|
37
41
|
subscribe: SubService<ProviderClient> | null;
|
|
42
|
+
/** @hidden */
|
|
38
43
|
router: Router<typeof this.stream> | null;
|
|
44
|
+
/** @hidden */
|
|
39
45
|
taskService: TaskService | null;
|
|
40
46
|
logger: ILogger;
|
|
41
47
|
cacheMode: CacheMode;
|
|
@@ -161,17 +167,27 @@ declare class EngineService {
|
|
|
161
167
|
*/
|
|
162
168
|
scrub(jobId: string): Promise<void>;
|
|
163
169
|
/**
|
|
170
|
+
* Delivers a signal (data payload) to a paused hook activity,
|
|
171
|
+
* resuming its Leg 2 execution. The `topic` must match a hook rule
|
|
172
|
+
* defined in the YAML graph's `hooks` section. The engine locates
|
|
173
|
+
* the target activity and dimension for reentry based on the hook
|
|
174
|
+
* rule's match conditions.
|
|
175
|
+
*
|
|
164
176
|
* @private
|
|
165
177
|
*/
|
|
166
|
-
|
|
178
|
+
signal(topic: string, data: JobData, status?: StreamStatus, code?: StreamCode, transaction?: ProviderTransaction): Promise<string>;
|
|
167
179
|
/**
|
|
168
180
|
* @private
|
|
169
181
|
*/
|
|
170
182
|
hookTime(jobId: string, gId: string, topicOrActivity: string, type?: WorkListTaskType): Promise<string | void>;
|
|
171
183
|
/**
|
|
184
|
+
* Fan-out variant of `signal()` that delivers data to **all**
|
|
185
|
+
* paused workflows matching a search query. Useful for resuming
|
|
186
|
+
* a batch of workflows waiting on the same external event.
|
|
187
|
+
*
|
|
172
188
|
* @private
|
|
173
189
|
*/
|
|
174
|
-
|
|
190
|
+
signalAll(hookTopic: string, data: JobData, keyResolver: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
|
|
175
191
|
/**
|
|
176
192
|
* @private
|
|
177
193
|
*/
|