@hotmeshio/hotmesh 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -142
- package/build/modules/enums.d.ts +7 -0
- package/build/modules/enums.js +16 -1
- package/build/modules/utils.d.ts +27 -0
- package/build/modules/utils.js +52 -1
- package/build/package.json +10 -8
- package/build/services/connector/providers/postgres.js +3 -0
- package/build/services/hotmesh/index.d.ts +66 -15
- package/build/services/hotmesh/index.js +84 -15
- package/build/services/memflow/index.d.ts +100 -14
- package/build/services/memflow/index.js +100 -14
- package/build/services/memflow/worker.d.ts +97 -0
- package/build/services/memflow/worker.js +217 -0
- package/build/services/memflow/workflow/proxyActivities.d.ts +74 -3
- package/build/services/memflow/workflow/proxyActivities.js +81 -4
- package/build/services/router/consumption/index.d.ts +2 -1
- package/build/services/router/consumption/index.js +38 -2
- package/build/services/router/error-handling/index.d.ts +3 -3
- package/build/services/router/error-handling/index.js +48 -13
- package/build/services/router/index.d.ts +1 -0
- package/build/services/router/index.js +2 -1
- package/build/services/store/index.d.ts +3 -2
- package/build/services/store/providers/postgres/kvtypes/hash/basic.js +36 -6
- package/build/services/store/providers/postgres/kvtypes/hash/expire.js +12 -2
- package/build/services/store/providers/postgres/kvtypes/hash/scan.js +30 -10
- package/build/services/store/providers/postgres/kvtypes/list.js +68 -10
- package/build/services/store/providers/postgres/kvtypes/string.js +60 -10
- package/build/services/store/providers/postgres/kvtypes/zset.js +92 -22
- package/build/services/store/providers/postgres/postgres.d.ts +3 -3
- package/build/services/store/providers/redis/_base.d.ts +3 -3
- package/build/services/store/providers/redis/ioredis.js +17 -7
- package/build/services/stream/providers/postgres/kvtables.js +76 -23
- package/build/services/stream/providers/postgres/lifecycle.d.ts +19 -0
- package/build/services/stream/providers/postgres/lifecycle.js +54 -0
- package/build/services/stream/providers/postgres/messages.d.ts +56 -0
- package/build/services/stream/providers/postgres/messages.js +253 -0
- package/build/services/stream/providers/postgres/notifications.d.ts +59 -0
- package/build/services/stream/providers/postgres/notifications.js +357 -0
- package/build/services/stream/providers/postgres/postgres.d.ts +110 -11
- package/build/services/stream/providers/postgres/postgres.js +196 -488
- package/build/services/stream/providers/postgres/scout.d.ts +68 -0
- package/build/services/stream/providers/postgres/scout.js +233 -0
- package/build/services/stream/providers/postgres/stats.d.ts +49 -0
- package/build/services/stream/providers/postgres/stats.js +113 -0
- package/build/services/sub/providers/postgres/postgres.js +37 -5
- package/build/services/sub/providers/redis/ioredis.js +13 -2
- package/build/services/sub/providers/redis/redis.js +13 -2
- package/build/services/worker/index.d.ts +1 -0
- package/build/services/worker/index.js +2 -0
- package/build/types/hotmesh.d.ts +42 -2
- package/build/types/index.d.ts +3 -3
- package/build/types/memflow.d.ts +32 -0
- package/build/types/provider.d.ts +16 -0
- package/build/types/stream.d.ts +92 -1
- package/package.json +10 -8
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { HotMesh } from '../hotmesh';
|
|
2
2
|
import { Connection, Registry, WorkerConfig, WorkerOptions } from '../../types/memflow';
|
|
3
|
+
import { StreamData, StreamDataResponse } from '../../types/stream';
|
|
3
4
|
/**
|
|
4
5
|
* The *Worker* service Registers worker functions and connects them to the mesh,
|
|
5
6
|
* using the target backend provider/s (Postgres, NATS, etc).
|
|
@@ -58,6 +59,102 @@ export declare class WorkerService {
|
|
|
58
59
|
* @private
|
|
59
60
|
*/
|
|
60
61
|
static registerActivities<ACT>(activities: ACT): Registry;
|
|
62
|
+
/**
|
|
63
|
+
* Register activity workers for a task queue. Activities are invoked via message queue,
|
|
64
|
+
* so they can run on different servers from workflows.
|
|
65
|
+
*
|
|
66
|
+
* The task queue name gets `-activity` appended automatically for the worker topic.
|
|
67
|
+
* For example, `taskQueue: 'payment'` creates a worker listening on `payment-activity`.
|
|
68
|
+
*
|
|
69
|
+
* @param config - Worker configuration (connection, namespace, taskQueue)
|
|
70
|
+
* @param activities - Activity functions to register
|
|
71
|
+
* @param activityTaskQueue - Task queue name (without `-activity` suffix).
|
|
72
|
+
* Defaults to `config.taskQueue` if not provided.
|
|
73
|
+
*
|
|
74
|
+
* @returns Promise<HotMesh> The initialized activity worker
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* ```typescript
|
|
78
|
+
* // Activity worker (can be on separate server)
|
|
79
|
+
* import { MemFlow } from '@hotmeshio/hotmesh';
|
|
80
|
+
* import { Client as Postgres } from 'pg';
|
|
81
|
+
*
|
|
82
|
+
* const activities = {
|
|
83
|
+
* async processPayment(amount: number): Promise<string> {
|
|
84
|
+
* return `Processed $${amount}`;
|
|
85
|
+
* },
|
|
86
|
+
* async sendEmail(to: string, subject: string): Promise<void> {
|
|
87
|
+
* // Send email
|
|
88
|
+
* }
|
|
89
|
+
* };
|
|
90
|
+
*
|
|
91
|
+
* await MemFlow.registerActivityWorker({
|
|
92
|
+
* connection: {
|
|
93
|
+
* class: Postgres,
|
|
94
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
|
|
95
|
+
* },
|
|
96
|
+
* taskQueue: 'payment' // Listens on 'payment-activity'
|
|
97
|
+
* }, activities, 'payment');
|
|
98
|
+
* ```
|
|
99
|
+
*
|
|
100
|
+
* @example
|
|
101
|
+
* ```typescript
|
|
102
|
+
* // Workflow worker (can be on different server)
|
|
103
|
+
* async function orderWorkflow(orderId: string, amount: number) {
|
|
104
|
+
* const { processPayment, sendEmail } = MemFlow.workflow.proxyActivities<{
|
|
105
|
+
* processPayment: (amount: number) => Promise<string>;
|
|
106
|
+
* sendEmail: (to: string, subject: string) => Promise<void>;
|
|
107
|
+
* }>({
|
|
108
|
+
* taskQueue: 'payment',
|
|
109
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
110
|
+
* });
|
|
111
|
+
*
|
|
112
|
+
* const result = await processPayment(amount);
|
|
113
|
+
* await sendEmail('customer@example.com', 'Order confirmed');
|
|
114
|
+
* return result;
|
|
115
|
+
* }
|
|
116
|
+
*
|
|
117
|
+
* await MemFlow.Worker.create({
|
|
118
|
+
* connection: {
|
|
119
|
+
* class: Postgres,
|
|
120
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
|
|
121
|
+
* },
|
|
122
|
+
* taskQueue: 'orders',
|
|
123
|
+
* workflow: orderWorkflow
|
|
124
|
+
* });
|
|
125
|
+
* ```
|
|
126
|
+
*
|
|
127
|
+
* @example
|
|
128
|
+
* ```typescript
|
|
129
|
+
* // Shared activity pool for interceptors
|
|
130
|
+
* await MemFlow.registerActivityWorker({
|
|
131
|
+
* connection: {
|
|
132
|
+
* class: Postgres,
|
|
133
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
|
|
134
|
+
* },
|
|
135
|
+
* taskQueue: 'shared'
|
|
136
|
+
* }, { auditLog, collectMetrics }, 'shared');
|
|
137
|
+
*
|
|
138
|
+
* const interceptor: WorkflowInterceptor = {
|
|
139
|
+
* async execute(ctx, next) {
|
|
140
|
+
* const { auditLog } = MemFlow.workflow.proxyActivities<{
|
|
141
|
+
* auditLog: (id: string, action: string) => Promise<void>;
|
|
142
|
+
* }>({
|
|
143
|
+
* taskQueue: 'shared',
|
|
144
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
145
|
+
* });
|
|
146
|
+
* await auditLog(ctx.get('workflowId'), 'started');
|
|
147
|
+
* return next();
|
|
148
|
+
* }
|
|
149
|
+
* };
|
|
150
|
+
* ```
|
|
151
|
+
*/
|
|
152
|
+
static registerActivityWorker(config: Partial<WorkerConfig>, activities: any, activityTaskQueue?: string): Promise<HotMesh>;
|
|
153
|
+
/**
|
|
154
|
+
* Create an activity callback function that can be used by activity workers
|
|
155
|
+
* @private
|
|
156
|
+
*/
|
|
157
|
+
static createActivityCallback(): (payload: StreamData) => Promise<StreamDataResponse>;
|
|
61
158
|
/**
|
|
62
159
|
* Connects a worker to the mesh.
|
|
63
160
|
*
|
|
@@ -104,6 +104,219 @@ class WorkerService {
|
|
|
104
104
|
}
|
|
105
105
|
return WorkerService.activityRegistry;
|
|
106
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Register activity workers for a task queue. Activities are invoked via message queue,
|
|
109
|
+
* so they can run on different servers from workflows.
|
|
110
|
+
*
|
|
111
|
+
* The task queue name gets `-activity` appended automatically for the worker topic.
|
|
112
|
+
* For example, `taskQueue: 'payment'` creates a worker listening on `payment-activity`.
|
|
113
|
+
*
|
|
114
|
+
* @param config - Worker configuration (connection, namespace, taskQueue)
|
|
115
|
+
* @param activities - Activity functions to register
|
|
116
|
+
* @param activityTaskQueue - Task queue name (without `-activity` suffix).
|
|
117
|
+
* Defaults to `config.taskQueue` if not provided.
|
|
118
|
+
*
|
|
119
|
+
* @returns Promise<HotMesh> The initialized activity worker
|
|
120
|
+
*
|
|
121
|
+
* @example
|
|
122
|
+
* ```typescript
|
|
123
|
+
* // Activity worker (can be on separate server)
|
|
124
|
+
* import { MemFlow } from '@hotmeshio/hotmesh';
|
|
125
|
+
* import { Client as Postgres } from 'pg';
|
|
126
|
+
*
|
|
127
|
+
* const activities = {
|
|
128
|
+
* async processPayment(amount: number): Promise<string> {
|
|
129
|
+
* return `Processed $${amount}`;
|
|
130
|
+
* },
|
|
131
|
+
* async sendEmail(to: string, subject: string): Promise<void> {
|
|
132
|
+
* // Send email
|
|
133
|
+
* }
|
|
134
|
+
* };
|
|
135
|
+
*
|
|
136
|
+
* await MemFlow.registerActivityWorker({
|
|
137
|
+
* connection: {
|
|
138
|
+
* class: Postgres,
|
|
139
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
|
|
140
|
+
* },
|
|
141
|
+
* taskQueue: 'payment' // Listens on 'payment-activity'
|
|
142
|
+
* }, activities, 'payment');
|
|
143
|
+
* ```
|
|
144
|
+
*
|
|
145
|
+
* @example
|
|
146
|
+
* ```typescript
|
|
147
|
+
* // Workflow worker (can be on different server)
|
|
148
|
+
* async function orderWorkflow(orderId: string, amount: number) {
|
|
149
|
+
* const { processPayment, sendEmail } = MemFlow.workflow.proxyActivities<{
|
|
150
|
+
* processPayment: (amount: number) => Promise<string>;
|
|
151
|
+
* sendEmail: (to: string, subject: string) => Promise<void>;
|
|
152
|
+
* }>({
|
|
153
|
+
* taskQueue: 'payment',
|
|
154
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
155
|
+
* });
|
|
156
|
+
*
|
|
157
|
+
* const result = await processPayment(amount);
|
|
158
|
+
* await sendEmail('customer@example.com', 'Order confirmed');
|
|
159
|
+
* return result;
|
|
160
|
+
* }
|
|
161
|
+
*
|
|
162
|
+
* await MemFlow.Worker.create({
|
|
163
|
+
* connection: {
|
|
164
|
+
* class: Postgres,
|
|
165
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
|
|
166
|
+
* },
|
|
167
|
+
* taskQueue: 'orders',
|
|
168
|
+
* workflow: orderWorkflow
|
|
169
|
+
* });
|
|
170
|
+
* ```
|
|
171
|
+
*
|
|
172
|
+
* @example
|
|
173
|
+
* ```typescript
|
|
174
|
+
* // Shared activity pool for interceptors
|
|
175
|
+
* await MemFlow.registerActivityWorker({
|
|
176
|
+
* connection: {
|
|
177
|
+
* class: Postgres,
|
|
178
|
+
* options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
|
|
179
|
+
* },
|
|
180
|
+
* taskQueue: 'shared'
|
|
181
|
+
* }, { auditLog, collectMetrics }, 'shared');
|
|
182
|
+
*
|
|
183
|
+
* const interceptor: WorkflowInterceptor = {
|
|
184
|
+
* async execute(ctx, next) {
|
|
185
|
+
* const { auditLog } = MemFlow.workflow.proxyActivities<{
|
|
186
|
+
* auditLog: (id: string, action: string) => Promise<void>;
|
|
187
|
+
* }>({
|
|
188
|
+
* taskQueue: 'shared',
|
|
189
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
190
|
+
* });
|
|
191
|
+
* await auditLog(ctx.get('workflowId'), 'started');
|
|
192
|
+
* return next();
|
|
193
|
+
* }
|
|
194
|
+
* };
|
|
195
|
+
* ```
|
|
196
|
+
*/
|
|
197
|
+
static async registerActivityWorker(config, activities, activityTaskQueue) {
|
|
198
|
+
// Register activities globally in the registry
|
|
199
|
+
WorkerService.registerActivities(activities);
|
|
200
|
+
// Use provided activityTaskQueue or fall back to config.taskQueue
|
|
201
|
+
const taskQueue = activityTaskQueue || config.taskQueue || 'memflow-activities';
|
|
202
|
+
// Append '-activity' suffix for the worker topic
|
|
203
|
+
const activityTopic = `${taskQueue}-activity`;
|
|
204
|
+
const targetNamespace = config?.namespace ?? factory_1.APP_ID;
|
|
205
|
+
const optionsHash = WorkerService.hashOptions(config?.connection);
|
|
206
|
+
const targetTopic = `${optionsHash}.${targetNamespace}.${activityTopic}`;
|
|
207
|
+
// Return existing worker if already initialized (idempotent)
|
|
208
|
+
if (WorkerService.instances.has(targetTopic)) {
|
|
209
|
+
return await WorkerService.instances.get(targetTopic);
|
|
210
|
+
}
|
|
211
|
+
// Create activity worker that listens on '{taskQueue}-activity' topic
|
|
212
|
+
const hotMeshWorker = await hotmesh_1.HotMesh.init({
|
|
213
|
+
guid: config.guid ? `${config.guid}XA` : undefined,
|
|
214
|
+
taskQueue,
|
|
215
|
+
logLevel: config.options?.logLevel ?? enums_1.HMSH_LOGLEVEL,
|
|
216
|
+
appId: targetNamespace,
|
|
217
|
+
engine: { connection: config.connection },
|
|
218
|
+
workers: [
|
|
219
|
+
{
|
|
220
|
+
topic: activityTopic,
|
|
221
|
+
connection: config.connection,
|
|
222
|
+
callback: WorkerService.createActivityCallback(),
|
|
223
|
+
},
|
|
224
|
+
],
|
|
225
|
+
});
|
|
226
|
+
WorkerService.instances.set(targetTopic, hotMeshWorker);
|
|
227
|
+
return hotMeshWorker;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Create an activity callback function that can be used by activity workers
|
|
231
|
+
* @private
|
|
232
|
+
*/
|
|
233
|
+
static createActivityCallback() {
|
|
234
|
+
return async (data) => {
|
|
235
|
+
try {
|
|
236
|
+
//always run the activity function when instructed; return the response
|
|
237
|
+
const activityInput = data.data;
|
|
238
|
+
const activityName = activityInput.activityName;
|
|
239
|
+
const activityFunction = WorkerService.activityRegistry[activityName];
|
|
240
|
+
if (!activityFunction) {
|
|
241
|
+
throw new Error(`Activity '${activityName}' not found in registry`);
|
|
242
|
+
}
|
|
243
|
+
const pojoResponse = await activityFunction.apply(null, activityInput.arguments);
|
|
244
|
+
return {
|
|
245
|
+
status: stream_1.StreamStatus.SUCCESS,
|
|
246
|
+
metadata: { ...data.metadata },
|
|
247
|
+
data: { response: pojoResponse },
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
catch (err) {
|
|
251
|
+
// Log error (note: we don't have access to this.activityRunner here)
|
|
252
|
+
console.error('memflow-worker-activity-err', {
|
|
253
|
+
name: err.name,
|
|
254
|
+
message: err.message,
|
|
255
|
+
stack: err.stack,
|
|
256
|
+
});
|
|
257
|
+
if (!(err instanceof errors_1.MemFlowTimeoutError) &&
|
|
258
|
+
!(err instanceof errors_1.MemFlowMaxedError) &&
|
|
259
|
+
!(err instanceof errors_1.MemFlowFatalError)) {
|
|
260
|
+
//use code 599 as a proxy for all retryable errors
|
|
261
|
+
// (basically anything not 596, 597, 598)
|
|
262
|
+
return {
|
|
263
|
+
status: stream_1.StreamStatus.SUCCESS,
|
|
264
|
+
code: 599,
|
|
265
|
+
metadata: { ...data.metadata },
|
|
266
|
+
data: {
|
|
267
|
+
$error: {
|
|
268
|
+
message: err.message,
|
|
269
|
+
stack: err.stack,
|
|
270
|
+
code: enums_1.HMSH_CODE_MEMFLOW_RETRYABLE,
|
|
271
|
+
},
|
|
272
|
+
},
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
else if (err instanceof errors_1.MemFlowTimeoutError) {
|
|
276
|
+
return {
|
|
277
|
+
status: stream_1.StreamStatus.SUCCESS,
|
|
278
|
+
code: 596,
|
|
279
|
+
metadata: { ...data.metadata },
|
|
280
|
+
data: {
|
|
281
|
+
$error: {
|
|
282
|
+
message: err.message,
|
|
283
|
+
stack: err.stack,
|
|
284
|
+
code: enums_1.HMSH_CODE_MEMFLOW_TIMEOUT,
|
|
285
|
+
},
|
|
286
|
+
},
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
else if (err instanceof errors_1.MemFlowMaxedError) {
|
|
290
|
+
return {
|
|
291
|
+
status: stream_1.StreamStatus.SUCCESS,
|
|
292
|
+
code: 597,
|
|
293
|
+
metadata: { ...data.metadata },
|
|
294
|
+
data: {
|
|
295
|
+
$error: {
|
|
296
|
+
message: err.message,
|
|
297
|
+
stack: err.stack,
|
|
298
|
+
code: enums_1.HMSH_CODE_MEMFLOW_MAXED,
|
|
299
|
+
},
|
|
300
|
+
},
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
else if (err instanceof errors_1.MemFlowFatalError) {
|
|
304
|
+
return {
|
|
305
|
+
status: stream_1.StreamStatus.SUCCESS,
|
|
306
|
+
code: 598,
|
|
307
|
+
metadata: { ...data.metadata },
|
|
308
|
+
data: {
|
|
309
|
+
$error: {
|
|
310
|
+
message: err.message,
|
|
311
|
+
stack: err.stack,
|
|
312
|
+
code: enums_1.HMSH_CODE_MEMFLOW_FATAL,
|
|
313
|
+
},
|
|
314
|
+
},
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
}
|
|
107
320
|
/**
|
|
108
321
|
* Connects a worker to the mesh.
|
|
109
322
|
*
|
|
@@ -173,6 +386,10 @@ class WorkerService {
|
|
|
173
386
|
const targetNamespace = config?.namespace ?? factory_1.APP_ID;
|
|
174
387
|
const optionsHash = WorkerService.hashOptions(config?.connection);
|
|
175
388
|
const targetTopic = `${optionsHash}.${targetNamespace}.${activityTopic}`;
|
|
389
|
+
// Return existing worker if already initialized
|
|
390
|
+
if (WorkerService.instances.has(targetTopic)) {
|
|
391
|
+
return await WorkerService.instances.get(targetTopic);
|
|
392
|
+
}
|
|
176
393
|
const hotMeshWorker = await hotmesh_1.HotMesh.init({
|
|
177
394
|
guid: config.guid ? `${config.guid}XA` : undefined,
|
|
178
395
|
taskQueue: config.taskQueue,
|
|
@@ -11,10 +11,81 @@ declare function getProxyInterruptPayload(context: ReturnType<typeof getContext>
|
|
|
11
11
|
*/
|
|
12
12
|
declare function wrapActivity<T>(activityName: string, options?: ActivityConfig): T;
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
14
|
+
* Create proxies for activity functions with automatic retry and deterministic replay.
|
|
15
|
+
* Activities execute via message queue, so they can run on different servers.
|
|
16
|
+
*
|
|
17
|
+
* Without `taskQueue`, activities use the workflow's task queue (e.g., `my-workflow-activity`).
|
|
18
|
+
* With `taskQueue`, activities use the specified queue (e.g., `payment-activity`).
|
|
19
|
+
*
|
|
20
|
+
* The `activities` parameter is optional. If activities are already registered via
|
|
21
|
+
* `registerActivityWorker()`, you can reference them by providing just the `taskQueue`
|
|
22
|
+
* and a TypeScript interface.
|
|
23
|
+
*
|
|
15
24
|
* @template ACT
|
|
16
|
-
* @param {ActivityConfig} [options] -
|
|
17
|
-
* @
|
|
25
|
+
* @param {ActivityConfig} [options] - Activity configuration
|
|
26
|
+
* @param {any} [options.activities] - (Optional) Activity functions to register inline
|
|
27
|
+
* @param {string} [options.taskQueue] - (Optional) Task queue name (without `-activity` suffix)
|
|
28
|
+
* @param {object} [options.retryPolicy] - Retry configuration
|
|
29
|
+
* @returns {ProxyType<ACT>} Proxy for calling activities with durability and retry
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```typescript
|
|
33
|
+
* // Inline registration (activities in same codebase)
|
|
34
|
+
* const activities = MemFlow.workflow.proxyActivities<typeof activities>({
|
|
35
|
+
* activities: { processData, validateData },
|
|
36
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
37
|
+
* });
|
|
38
|
+
*
|
|
39
|
+
* await activities.processData('input');
|
|
40
|
+
* ```
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* ```typescript
|
|
44
|
+
* // Reference pre-registered activities (can be on different server)
|
|
45
|
+
* interface PaymentActivities {
|
|
46
|
+
* processPayment: (amount: number) => Promise<string>;
|
|
47
|
+
* sendEmail: (to: string, subject: string) => Promise<void>;
|
|
48
|
+
* }
|
|
49
|
+
*
|
|
50
|
+
* const { processPayment, sendEmail } =
|
|
51
|
+
* MemFlow.workflow.proxyActivities<PaymentActivities>({
|
|
52
|
+
* taskQueue: 'payment',
|
|
53
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
54
|
+
* });
|
|
55
|
+
*
|
|
56
|
+
* const result = await processPayment(100.00);
|
|
57
|
+
* await sendEmail('user@example.com', 'Payment processed');
|
|
58
|
+
* ```
|
|
59
|
+
*
|
|
60
|
+
* @example
|
|
61
|
+
* ```typescript
|
|
62
|
+
* // Shared activities in interceptor
|
|
63
|
+
* const interceptor: WorkflowInterceptor = {
|
|
64
|
+
* async execute(ctx, next) {
|
|
65
|
+
* const { auditLog } = MemFlow.workflow.proxyActivities<{
|
|
66
|
+
* auditLog: (id: string, action: string) => Promise<void>;
|
|
67
|
+
* }>({
|
|
68
|
+
* taskQueue: 'shared',
|
|
69
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
70
|
+
* });
|
|
71
|
+
*
|
|
72
|
+
* await auditLog(ctx.get('workflowId'), 'started');
|
|
73
|
+
* const result = await next();
|
|
74
|
+
* await auditLog(ctx.get('workflowId'), 'completed');
|
|
75
|
+
* return result;
|
|
76
|
+
* }
|
|
77
|
+
* };
|
|
78
|
+
* ```
|
|
79
|
+
*
|
|
80
|
+
* @example
|
|
81
|
+
* ```typescript
|
|
82
|
+
* // Custom task queue for specific activities
|
|
83
|
+
* const highPriority = MemFlow.workflow.proxyActivities<typeof activities>({
|
|
84
|
+
* activities: { criticalProcess },
|
|
85
|
+
* taskQueue: 'high-priority',
|
|
86
|
+
* retryPolicy: { maximumAttempts: 5 }
|
|
87
|
+
* });
|
|
88
|
+
* ```
|
|
18
89
|
*/
|
|
19
90
|
export declare function proxyActivities<ACT>(options?: ActivityConfig): ProxyType<ACT>;
|
|
20
91
|
export { wrapActivity, getProxyInterruptPayload };
|
|
@@ -10,7 +10,11 @@ const didRun_1 = require("./didRun");
|
|
|
10
10
|
*/
|
|
11
11
|
function getProxyInterruptPayload(context, activityName, execIndex, args, options) {
|
|
12
12
|
const { workflowDimension, workflowId, originJobId, workflowTopic, expire } = context;
|
|
13
|
-
|
|
13
|
+
// Use explicitly provided taskQueue, otherwise derive from workflow (original behavior)
|
|
14
|
+
// This keeps backward compatibility while allowing explicit global/custom queues
|
|
15
|
+
const activityTopic = options?.taskQueue
|
|
16
|
+
? `${options.taskQueue}-activity`
|
|
17
|
+
: `${workflowTopic}-activity`;
|
|
14
18
|
const activityJobId = `-${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
|
|
15
19
|
let maximumInterval;
|
|
16
20
|
if (options?.retryPolicy?.maximumInterval) {
|
|
@@ -77,15 +81,88 @@ function wrapActivity(activityName, options) {
|
|
|
77
81
|
}
|
|
78
82
|
exports.wrapActivity = wrapActivity;
|
|
79
83
|
/**
|
|
80
|
-
*
|
|
84
|
+
* Create proxies for activity functions with automatic retry and deterministic replay.
|
|
85
|
+
* Activities execute via message queue, so they can run on different servers.
|
|
86
|
+
*
|
|
87
|
+
* Without `taskQueue`, activities use the workflow's task queue (e.g., `my-workflow-activity`).
|
|
88
|
+
* With `taskQueue`, activities use the specified queue (e.g., `payment-activity`).
|
|
89
|
+
*
|
|
90
|
+
* The `activities` parameter is optional. If activities are already registered via
|
|
91
|
+
* `registerActivityWorker()`, you can reference them by providing just the `taskQueue`
|
|
92
|
+
* and a TypeScript interface.
|
|
93
|
+
*
|
|
81
94
|
* @template ACT
|
|
82
|
-
* @param {ActivityConfig} [options] -
|
|
83
|
-
* @
|
|
95
|
+
* @param {ActivityConfig} [options] - Activity configuration
|
|
96
|
+
* @param {any} [options.activities] - (Optional) Activity functions to register inline
|
|
97
|
+
* @param {string} [options.taskQueue] - (Optional) Task queue name (without `-activity` suffix)
|
|
98
|
+
* @param {object} [options.retryPolicy] - Retry configuration
|
|
99
|
+
* @returns {ProxyType<ACT>} Proxy for calling activities with durability and retry
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* ```typescript
|
|
103
|
+
* // Inline registration (activities in same codebase)
|
|
104
|
+
* const activities = MemFlow.workflow.proxyActivities<typeof activities>({
|
|
105
|
+
* activities: { processData, validateData },
|
|
106
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
107
|
+
* });
|
|
108
|
+
*
|
|
109
|
+
* await activities.processData('input');
|
|
110
|
+
* ```
|
|
111
|
+
*
|
|
112
|
+
* @example
|
|
113
|
+
* ```typescript
|
|
114
|
+
* // Reference pre-registered activities (can be on different server)
|
|
115
|
+
* interface PaymentActivities {
|
|
116
|
+
* processPayment: (amount: number) => Promise<string>;
|
|
117
|
+
* sendEmail: (to: string, subject: string) => Promise<void>;
|
|
118
|
+
* }
|
|
119
|
+
*
|
|
120
|
+
* const { processPayment, sendEmail } =
|
|
121
|
+
* MemFlow.workflow.proxyActivities<PaymentActivities>({
|
|
122
|
+
* taskQueue: 'payment',
|
|
123
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
124
|
+
* });
|
|
125
|
+
*
|
|
126
|
+
* const result = await processPayment(100.00);
|
|
127
|
+
* await sendEmail('user@example.com', 'Payment processed');
|
|
128
|
+
* ```
|
|
129
|
+
*
|
|
130
|
+
* @example
|
|
131
|
+
* ```typescript
|
|
132
|
+
* // Shared activities in interceptor
|
|
133
|
+
* const interceptor: WorkflowInterceptor = {
|
|
134
|
+
* async execute(ctx, next) {
|
|
135
|
+
* const { auditLog } = MemFlow.workflow.proxyActivities<{
|
|
136
|
+
* auditLog: (id: string, action: string) => Promise<void>;
|
|
137
|
+
* }>({
|
|
138
|
+
* taskQueue: 'shared',
|
|
139
|
+
* retryPolicy: { maximumAttempts: 3 }
|
|
140
|
+
* });
|
|
141
|
+
*
|
|
142
|
+
* await auditLog(ctx.get('workflowId'), 'started');
|
|
143
|
+
* const result = await next();
|
|
144
|
+
* await auditLog(ctx.get('workflowId'), 'completed');
|
|
145
|
+
* return result;
|
|
146
|
+
* }
|
|
147
|
+
* };
|
|
148
|
+
* ```
|
|
149
|
+
*
|
|
150
|
+
* @example
|
|
151
|
+
* ```typescript
|
|
152
|
+
* // Custom task queue for specific activities
|
|
153
|
+
* const highPriority = MemFlow.workflow.proxyActivities<typeof activities>({
|
|
154
|
+
* activities: { criticalProcess },
|
|
155
|
+
* taskQueue: 'high-priority',
|
|
156
|
+
* retryPolicy: { maximumAttempts: 5 }
|
|
157
|
+
* });
|
|
158
|
+
* ```
|
|
84
159
|
*/
|
|
85
160
|
function proxyActivities(options) {
|
|
161
|
+
// Register activities if provided (optional - may already be registered remotely)
|
|
86
162
|
if (options?.activities) {
|
|
87
163
|
common_1.WorkerService.registerActivities(options.activities);
|
|
88
164
|
}
|
|
165
|
+
// Create proxy for all registered activities
|
|
89
166
|
const proxy = {};
|
|
90
167
|
const keys = Object.keys(common_1.WorkerService.activityRegistry);
|
|
91
168
|
if (keys.length) {
|
|
@@ -19,7 +19,8 @@ export declare class ConsumptionManager<S extends StreamService<ProviderClient,
|
|
|
19
19
|
private counts;
|
|
20
20
|
private hasReachedMaxBackoff;
|
|
21
21
|
private router;
|
|
22
|
-
|
|
22
|
+
private retryPolicy;
|
|
23
|
+
constructor(stream: S, logger: ILogger, throttleManager: ThrottleManager, errorHandler: ErrorHandler, lifecycleManager: LifecycleManager<S>, reclaimDelay: number, reclaimCount: number, appId: string, role: any, router: any, retryPolicy?: import('../../../types/stream').RetryPolicy);
|
|
23
24
|
createGroup(stream: string, group: string): Promise<void>;
|
|
24
25
|
publishMessage(topic: string, streamData: StreamData | StreamDataResponse, transaction?: ProviderTransaction): Promise<string | ProviderTransaction>;
|
|
25
26
|
consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
|
|
@@ -7,7 +7,7 @@ const config_1 = require("../config");
|
|
|
7
7
|
const stream_1 = require("../../../types/stream");
|
|
8
8
|
const key_1 = require("../../../modules/key");
|
|
9
9
|
class ConsumptionManager {
|
|
10
|
-
constructor(stream, logger, throttleManager, errorHandler, lifecycleManager, reclaimDelay, reclaimCount, appId, role, router) {
|
|
10
|
+
constructor(stream, logger, throttleManager, errorHandler, lifecycleManager, reclaimDelay, reclaimCount, appId, role, router, retryPolicy) {
|
|
11
11
|
this.errorCount = 0;
|
|
12
12
|
this.counts = {};
|
|
13
13
|
this.stream = stream;
|
|
@@ -20,6 +20,7 @@ class ConsumptionManager {
|
|
|
20
20
|
this.appId = appId;
|
|
21
21
|
this.role = role;
|
|
22
22
|
this.router = router;
|
|
23
|
+
this.retryPolicy = retryPolicy;
|
|
23
24
|
}
|
|
24
25
|
async createGroup(stream, group) {
|
|
25
26
|
try {
|
|
@@ -32,6 +33,31 @@ class ConsumptionManager {
|
|
|
32
33
|
async publishMessage(topic, streamData, transaction) {
|
|
33
34
|
const code = streamData?.code || '200';
|
|
34
35
|
this.counts[code] = (this.counts[code] || 0) + 1;
|
|
36
|
+
// Extract retry policy from child workflow (590) and activity (591) message data
|
|
37
|
+
// ONLY if values differ from YAML defaults (10, 3/5, 120)
|
|
38
|
+
// If they're defaults, let old retry mechanism (policies.retry) handle it
|
|
39
|
+
const codeNum = typeof code === 'number' ? code : parseInt(code, 10);
|
|
40
|
+
if ((codeNum === 590 || codeNum === 591) && streamData.data) {
|
|
41
|
+
const data = streamData.data;
|
|
42
|
+
const backoff = data.backoffCoefficient;
|
|
43
|
+
const attempts = data.maximumAttempts;
|
|
44
|
+
const maxInterval = typeof data.maximumInterval === 'string'
|
|
45
|
+
? parseInt(data.maximumInterval)
|
|
46
|
+
: data.maximumInterval;
|
|
47
|
+
// Only extract if values are NOT the YAML defaults
|
|
48
|
+
// YAML defaults: backoffCoefficient=10, maximumAttempts=3 or 5, maximumInterval=120
|
|
49
|
+
const hasNonDefaultBackoff = backoff != null && backoff !== 10;
|
|
50
|
+
const hasNonDefaultAttempts = attempts != null && attempts !== 3 && attempts !== 5;
|
|
51
|
+
const hasNonDefaultInterval = maxInterval != null && maxInterval !== 120;
|
|
52
|
+
if (hasNonDefaultBackoff || hasNonDefaultAttempts || hasNonDefaultInterval) {
|
|
53
|
+
// Has custom values from config - add _streamRetryConfig
|
|
54
|
+
streamData._streamRetryConfig = {
|
|
55
|
+
max_retry_attempts: attempts,
|
|
56
|
+
backoff_coefficient: backoff,
|
|
57
|
+
maximum_interval_seconds: maxInterval,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
}
|
|
35
61
|
const stream = this.stream.mintKey(key_1.KeyType.STREAMS, { topic });
|
|
36
62
|
const responses = await this.stream.publishMessages(stream, [JSON.stringify(streamData)], { transaction });
|
|
37
63
|
return responses[0];
|
|
@@ -390,7 +416,17 @@ class ConsumptionManager {
|
|
|
390
416
|
async publishResponse(input, output) {
|
|
391
417
|
if (output && typeof output === 'object') {
|
|
392
418
|
if (output.status === 'error') {
|
|
393
|
-
|
|
419
|
+
// Extract retry policy with priority:
|
|
420
|
+
// 1. Use message-level _streamRetryConfig (from database columns or previous retry)
|
|
421
|
+
// 2. Fall back to router-level retryPolicy (from worker config)
|
|
422
|
+
const retryPolicy = input._streamRetryConfig
|
|
423
|
+
? {
|
|
424
|
+
maximumAttempts: input._streamRetryConfig.max_retry_attempts,
|
|
425
|
+
backoffCoefficient: input._streamRetryConfig.backoff_coefficient,
|
|
426
|
+
maximumInterval: input._streamRetryConfig.maximum_interval_seconds,
|
|
427
|
+
}
|
|
428
|
+
: this.retryPolicy;
|
|
429
|
+
return await this.errorHandler.handleRetry(input, output, this.publishMessage.bind(this), retryPolicy);
|
|
394
430
|
}
|
|
395
431
|
else if (typeof output.metadata !== 'object') {
|
|
396
432
|
output.metadata = { ...input.metadata, guid: (0, utils_1.guid)() };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { StreamData, StreamDataResponse } from '../../../types/stream';
|
|
1
|
+
import { StreamData, StreamDataResponse, RetryPolicy } from '../../../types/stream';
|
|
2
2
|
export declare class ErrorHandler {
|
|
3
|
-
shouldRetry(input: StreamData, output: StreamDataResponse): [boolean, number];
|
|
3
|
+
shouldRetry(input: StreamData, output: StreamDataResponse, retryPolicy?: RetryPolicy): [boolean, number];
|
|
4
4
|
structureUnhandledError(input: StreamData, err: Error): StreamDataResponse;
|
|
5
5
|
structureUnacknowledgedError(input: StreamData): StreamDataResponse;
|
|
6
6
|
structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse;
|
|
7
|
-
handleRetry(input: StreamData, output: StreamDataResponse, publishMessage: (topic: string, streamData: StreamData | StreamDataResponse) => Promise<string
|
|
7
|
+
handleRetry(input: StreamData, output: StreamDataResponse, publishMessage: (topic: string, streamData: StreamData | StreamDataResponse) => Promise<string>, retryPolicy?: RetryPolicy): Promise<string>;
|
|
8
8
|
}
|