@hotmeshio/hotmesh 0.0.11 → 0.0.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +6 -6
  2. package/build/modules/errors.d.ts +22 -1
  3. package/build/modules/errors.js +28 -1
  4. package/build/modules/utils.d.ts +2 -1
  5. package/build/modules/utils.js +5 -1
  6. package/build/package.json +8 -3
  7. package/build/services/activities/activity.d.ts +2 -0
  8. package/build/services/activities/activity.js +16 -10
  9. package/build/services/activities/await.d.ts +2 -6
  10. package/build/services/activities/await.js +12 -75
  11. package/build/services/activities/cycle.js +4 -2
  12. package/build/services/activities/index.d.ts +2 -2
  13. package/build/services/activities/index.js +2 -2
  14. package/build/services/activities/signal.d.ts +16 -0
  15. package/build/services/activities/signal.js +94 -0
  16. package/build/services/activities/trigger.js +6 -5
  17. package/build/services/activities/worker.d.ts +2 -1
  18. package/build/services/activities/worker.js +11 -6
  19. package/build/services/compiler/deployer.js +3 -1
  20. package/build/services/connector/index.js +3 -3
  21. package/build/services/durable/client.d.ts +4 -3
  22. package/build/services/durable/client.js +50 -23
  23. package/build/services/durable/connection.js +2 -2
  24. package/build/services/durable/factory.d.ts +22 -18
  25. package/build/services/durable/factory.js +722 -50
  26. package/build/services/durable/handle.d.ts +1 -0
  27. package/build/services/durable/handle.js +5 -1
  28. package/build/services/durable/worker.d.ts +3 -8
  29. package/build/services/durable/worker.js +75 -73
  30. package/build/services/durable/workflow.d.ts +5 -0
  31. package/build/services/durable/workflow.js +93 -24
  32. package/build/services/engine/index.d.ts +5 -5
  33. package/build/services/engine/index.js +24 -14
  34. package/build/services/hotmesh/index.d.ts +1 -0
  35. package/build/services/hotmesh/index.js +5 -3
  36. package/build/services/mapper/index.js +1 -1
  37. package/build/services/pipe/functions/array.d.ts +1 -0
  38. package/build/services/pipe/functions/array.js +3 -0
  39. package/build/services/reporter/index.js +9 -2
  40. package/build/services/signaler/store.js +8 -3
  41. package/build/services/signaler/stream.js +3 -3
  42. package/build/services/store/clients/ioredis.js +15 -15
  43. package/build/services/store/clients/redis.js +18 -18
  44. package/build/services/store/index.d.ts +1 -1
  45. package/build/services/store/index.js +11 -3
  46. package/build/services/task/index.js +3 -3
  47. package/build/types/activity.d.ts +15 -6
  48. package/build/types/durable.d.ts +15 -2
  49. package/build/types/index.d.ts +1 -1
  50. package/build/types/stats.d.ts +1 -0
  51. package/modules/errors.ts +35 -0
  52. package/modules/utils.ts +5 -1
  53. package/package.json +8 -3
  54. package/services/activities/activity.ts +19 -9
  55. package/services/activities/await.ts +14 -90
  56. package/services/activities/cycle.ts +4 -2
  57. package/services/activities/index.ts +2 -2
  58. package/services/activities/signal.ts +124 -0
  59. package/services/activities/trigger.ts +6 -5
  60. package/services/activities/worker.ts +13 -13
  61. package/services/compiler/deployer.ts +3 -1
  62. package/services/connector/index.ts +3 -3
  63. package/services/durable/client.ts +62 -24
  64. package/services/durable/connection.ts +2 -2
  65. package/services/durable/factory.ts +723 -49
  66. package/services/durable/handle.ts +6 -1
  67. package/services/durable/worker.ts +92 -79
  68. package/services/durable/workflow.ts +95 -25
  69. package/services/engine/index.ts +31 -22
  70. package/services/hotmesh/index.ts +8 -5
  71. package/services/mapper/index.ts +1 -1
  72. package/services/pipe/functions/array.ts +4 -0
  73. package/services/reporter/index.ts +10 -2
  74. package/services/signaler/store.ts +8 -3
  75. package/services/signaler/stream.ts +3 -3
  76. package/services/store/clients/ioredis.ts +15 -15
  77. package/services/store/clients/redis.ts +18 -18
  78. package/services/store/index.ts +12 -3
  79. package/services/task/index.ts +3 -3
  80. package/types/activity.ts +16 -7
  81. package/types/durable.ts +17 -1
  82. package/types/index.ts +1 -1
  83. package/types/stats.ts +1 -0
  84. package/build/services/activities/emit.d.ts +0 -9
  85. package/build/services/activities/emit.js +0 -13
  86. package/services/activities/emit.ts +0 -25
@@ -1,5 +1,6 @@
1
1
  import { GetStateError } from '../../modules/errors';
2
2
  import { Activity } from './activity';
3
+ import { CollatorService } from '../collator';
3
4
  import { EngineService } from '../engine';
4
5
  import {
5
6
  ActivityData,
@@ -7,15 +8,10 @@ import {
7
8
  AwaitActivity,
8
9
  ActivityType } from '../../types/activity';
9
10
  import { JobState } from '../../types/job';
10
- import { MultiResponseFlags } from '../../types/redis';
11
- import { StringScalarType } from '../../types/serializer';
12
- import {
13
- StreamCode,
14
- StreamData,
15
- StreamDataType,
16
- StreamStatus } from '../../types/stream';
11
+ import { MultiResponseFlags, RedisMulti } from '../../types/redis';
12
+ import { StreamData, StreamDataType } from '../../types/stream';
17
13
  import { TelemetryService } from '../telemetry';
18
- import { CollatorService } from '../collator';
14
+ import { Pipe } from '../pipe';
19
15
 
20
16
  class Await extends Activity {
21
17
  config: AwaitActivity;
@@ -35,24 +31,26 @@ class Await extends Activity {
35
31
  this.logger.debug('await-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
36
32
  let telemetry: TelemetryService;
37
33
  try {
34
+ //confirm entry is allowed and restore state
38
35
  this.setLeg(1);
39
36
  await CollatorService.notarizeEntry(this);
40
-
41
37
  await this.getState();
42
38
  telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
43
39
  telemetry.startActivitySpan(this.leg);
44
40
  this.mapInputData();
45
41
 
42
+ //save state and authorize reentry
46
43
  const multi = this.store.getMulti();
47
44
  //todo: await this.registerTimeout();
45
+ const messageId = await this.execActivity(multi);
48
46
  await CollatorService.authorizeReentry(this, multi);
49
47
  await this.setState(multi);
50
48
  await this.setStatus(0, multi);
51
49
  const multiResponse = await multi.exec() as MultiResponseFlags;
52
50
 
51
+ //telemetry
53
52
  telemetry.mapActivityAttributes();
54
53
  const jobStatus = this.resolveStatus(multiResponse);
55
- const messageId = await this.execActivity();
56
54
  telemetry.setActivityAttributes({
57
55
  'app.activity.mid': messageId,
58
56
  'app.job.jss': jobStatus
@@ -61,9 +59,9 @@ class Await extends Activity {
61
59
  } catch (error) {
62
60
  telemetry.setActivityError(error.message);
63
61
  if (error instanceof GetStateError) {
64
- this.logger.error('await-get-state-error', error);
62
+ this.logger.error('await-get-state-error', { error });
65
63
  } else {
66
- this.logger.error('await-process-error', error);
64
+ this.logger.error('await-process-error', { error });
67
65
  }
68
66
  throw error;
69
67
  } finally {
@@ -72,14 +70,14 @@ class Await extends Activity {
72
70
  }
73
71
  }
74
72
 
75
-
76
- async execActivity(): Promise<string> {
73
+ async execActivity(multi: RedisMulti): Promise<string> {
74
+ const topic = Pipe.resolve(this.config.subtype, this.context);
77
75
  const streamData: StreamData = {
78
76
  metadata: {
79
77
  jid: this.context.metadata.jid,
80
78
  dad: this.metadata.dad,
81
79
  aid: this.metadata.aid,
82
- topic: this.config.subtype,
80
+ topic,
83
81
  spn: this.context['$self'].output.metadata?.l1s,
84
82
  trc: this.context.metadata.trc,
85
83
  },
@@ -91,81 +89,7 @@ class Await extends Activity {
91
89
  retry: this.config.retry
92
90
  };
93
91
  }
94
- return (await this.engine.streamSignaler?.publishMessage(null, streamData)) as string;
95
- }
96
-
97
-
98
- //******** `RESOLVE` ENTRY POINT (B) ********//
99
- //this method is invoked when the job spawned by this job ends;
100
- //`this.data` is the job data produced by the spawned job
101
- async processEvent(status: StreamStatus = StreamStatus.SUCCESS, code: StreamCode = 200): Promise<void> {
102
- this.setLeg(2);
103
- const jid = this.context.metadata.jid;
104
- const aid = this.metadata.aid;
105
- if (!jid) {
106
- throw new Error('await-process-event-error');
107
- }
108
- this.logger.debug('await-resolve-await', { jid, aid, status, code });
109
- this.status = status;
110
- this.code = code;
111
- let telemetry: TelemetryService;
112
- try {
113
- await this.getState();
114
- const aState = await CollatorService.notarizeReentry(this);
115
- this.adjacentIndex = CollatorService.getDimensionalIndex(aState);
116
-
117
- telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
118
- telemetry.startActivitySpan(this.leg);
119
-
120
- let multiResponse: MultiResponseFlags = [];
121
- if (status === StreamStatus.SUCCESS) {
122
- this.bindActivityData('output');
123
- this.adjacencyList = await this.filterAdjacent();
124
- multiResponse = await this.processSuccessResponse(this.adjacencyList);
125
- } else {
126
- this.bindActivityError(this.data);
127
- this.adjacencyList = await this.filterAdjacent();
128
- multiResponse = await this.processErrorResponse(this.adjacencyList);
129
- }
130
-
131
- telemetry.mapActivityAttributes();
132
- const jobStatus = this.resolveStatus(multiResponse);
133
- const attrs: StringScalarType = { 'app.job.jss': jobStatus };
134
- const messageIds = await this.transition(this.adjacencyList, jobStatus);
135
- if (messageIds.length) {
136
- attrs['app.activity.mids'] = messageIds.join(',')
137
- }
138
- telemetry.setActivityAttributes(attrs);
139
- } catch (error) {
140
- this.logger.error('await-resolve-await-error', error);
141
- telemetry.setActivityError(error.message);
142
- throw error;
143
- } finally {
144
- telemetry.endActivitySpan();
145
- this.logger.debug('await-resolve-await-end', { jid, aid, status, code });
146
- }
147
- }
148
-
149
- async processSuccessResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags> {
150
- this.mapJobData();
151
- const multi = this.store.getMulti();
152
- await this.setState(multi);
153
- await CollatorService.notarizeCompletion(this, multi);
154
-
155
- await this.setStatus(adjacencyList.length - 1, multi);
156
- return await multi.exec() as MultiResponseFlags;
157
- }
158
-
159
- async processErrorResponse(adjacencyList: StreamData[]): Promise<MultiResponseFlags> {
160
- //todo: if adjacencyList.length == 0, then map to the job output
161
- // this method would be added to Base activity class
162
- //this.mapJobData();
163
- const multi = this.store.getMulti();
164
- await this.setState(multi);
165
- await CollatorService.notarizeCompletion(this, multi);
166
-
167
- await this.setStatus(adjacencyList.length - 1, multi);
168
- return await multi.exec() as MultiResponseFlags;
92
+ return (await this.engine.streamSignaler?.publishMessage(null, streamData, multi)) as string;
169
93
  }
170
94
  }
171
95
 
@@ -61,9 +61,9 @@ class Cycle extends Activity {
61
61
  return this.context.metadata.aid;
62
62
  } catch (error) {
63
63
  if (error instanceof GetStateError) {
64
- this.logger.error('cycle-get-state-error', error);
64
+ this.logger.error('cycle-get-state-error', { error });
65
65
  } else {
66
- this.logger.error('cycle-process-error', error);
66
+ this.logger.error('cycle-process-error', { error });
67
67
  }
68
68
  telemetry.setActivityError(error.message);
69
69
  throw error;
@@ -91,6 +91,8 @@ class Cycle extends Activity {
91
91
  dad: CollatorService.resolveReentryDimension(this),
92
92
  jid: this.context.metadata.jid,
93
93
  aid: this.config.ancestor,
94
+ spn: this.context['$self'].output.metadata?.l1s,
95
+ trc: this.context.metadata.trc,
94
96
  },
95
97
  data: this.context.data
96
98
  };
@@ -1,8 +1,8 @@
1
1
  import { Activity } from './activity';
2
2
  import { Await } from './await';
3
3
  import { Cycle } from './cycle';
4
- import { Emit } from './emit';
5
4
  import { Iterate } from './iterate';
5
+ import { Signal } from './signal';
6
6
  import { Trigger } from './trigger';
7
7
  import { Worker } from './worker';
8
8
 
@@ -11,7 +11,7 @@ export default {
11
11
  await: Await,
12
12
  cycle: Cycle,
13
13
  iterate: Iterate,
14
- emit: Emit,
14
+ signal: Signal,
15
15
  trigger: Trigger,
16
16
  worker: Worker,
17
17
  };
@@ -0,0 +1,124 @@
1
+ import { GetStateError } from '../../modules/errors';
2
+ import { Activity, ActivityType } from './activity';
3
+ import { CollatorService } from '../collator';
4
+ import { EngineService } from '../engine';
5
+ import { MapperService } from '../mapper';
6
+ import { Pipe } from '../pipe';
7
+ import { TelemetryService } from '../telemetry';
8
+ import {
9
+ ActivityData,
10
+ ActivityMetadata,
11
+ SignalActivity } from '../../types/activity';
12
+ import { JobState } from '../../types/job';
13
+ import { MultiResponseFlags, RedisMulti } from '../../types/redis';
14
+ import { StringScalarType } from '../../types/serializer';
15
+ import { JobStatsInput } from '../../types/stats';
16
+
17
+ class Signal extends Activity {
18
+ config: SignalActivity;
19
+
20
+ constructor(
21
+ config: ActivityType,
22
+ data: ActivityData,
23
+ metadata: ActivityMetadata,
24
+ hook: ActivityData | null,
25
+ engine: EngineService,
26
+ context?: JobState) {
27
+ super(config, data, metadata, hook, engine, context);
28
+ }
29
+
30
+
31
+ //******** LEG 1 ENTRY ********//
32
+ async process(): Promise<string> {
33
+ this.logger.debug('signal-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
34
+ let telemetry: TelemetryService;
35
+ try {
36
+ //verify entry is allowed
37
+ this.setLeg(1);
38
+ await CollatorService.notarizeEntry(this);
39
+ await this.getState();
40
+ telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
41
+ telemetry.startActivitySpan(this.leg);
42
+
43
+ //save state and notarize early completion (signals only run leg1)
44
+ const multi = this.store.getMulti();
45
+ this.adjacencyList = await this.filterAdjacent();
46
+ this.mapOutputData();
47
+ this.mapJobData();
48
+ await this.setState(multi);
49
+ await CollatorService.notarizeEarlyCompletion(this, multi);
50
+ await this.setStatus(this.adjacencyList.length - 1, multi);
51
+ const multiResponse = await multi.exec() as MultiResponseFlags;
52
+
53
+ //signal to awaken all paused jobs that share the targeted job key
54
+ await this.hookAll();
55
+
56
+ //transition to adjacent activities
57
+ const jobStatus = this.resolveStatus(multiResponse);
58
+ const attrs: StringScalarType = { 'app.job.jss': jobStatus };
59
+ const messageIds = await this.transition(this.adjacencyList, jobStatus);
60
+ if (messageIds.length) {
61
+ attrs['app.activity.mids'] = messageIds.join(',')
62
+ }
63
+ telemetry.mapActivityAttributes();
64
+ telemetry.setActivityAttributes(attrs);
65
+
66
+ return this.context.metadata.aid;
67
+ } catch (error) {
68
+ if (error instanceof GetStateError) {
69
+ this.logger.error('signal-get-state-error', { error });
70
+ } else {
71
+ this.logger.error('signal-process-error', { error });
72
+ }
73
+ telemetry.setActivityError(error.message);
74
+ throw error;
75
+ } finally {
76
+ telemetry.endActivitySpan();
77
+ this.logger.debug('signal-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
78
+ }
79
+ }
80
+
81
+ mapSignalData(): Record<string, any> {
82
+ if(this.config.signal?.maps) {
83
+ const mapper = new MapperService(this.config.signal.maps, this.context);
84
+ return mapper.mapRules();
85
+ }
86
+ }
87
+
88
+ mapResolverData(): Record<string, any> {
89
+ if(this.config.resolver?.maps) {
90
+ const mapper = new MapperService(this.config.resolver.maps, this.context);
91
+ return mapper.mapRules();
92
+ }
93
+ }
94
+
95
+ /**
96
+ * The signal activity will hook all paused jobs that share the same job key.
97
+ */
98
+ async hookAll(): Promise<string[]> {
99
+ //prep 1) generate `input signal data` (essentially the webhook payload)
100
+ const signalInputData = this.mapSignalData();
101
+
102
+ //prep 2) generate data that resolves the job key (per the YAML config)
103
+ const keyResolverData = this.mapResolverData() as JobStatsInput;
104
+ if (this.config.scrub) {
105
+ //self-clean the indexes upon use if configured
106
+ keyResolverData.scrub = true;
107
+ }
108
+
109
+ //prep 3) jobKeys can contain multiple indexes (per the YAML config)
110
+ const key_name = Pipe.resolve(this.config.key_name, this.context);
111
+ const key_value = Pipe.resolve(this.config.key_value, this.context);
112
+ const indexQueryFacets = [`${key_name}:${key_value}`];
113
+
114
+ //execute: `hookAll` will now resume all paused jobs that share the same job key
115
+ return await this.engine.hookAll(
116
+ this.config.topic,
117
+ signalInputData,
118
+ keyResolverData,
119
+ indexQueryFacets
120
+ );
121
+ }
122
+ }
123
+
124
+ export { Signal };
@@ -1,4 +1,4 @@
1
- import { v4 as uuidv4 } from 'uuid';
1
+ import { nanoid } from 'nanoid';
2
2
  import { DuplicateJobError } from '../../modules/errors';
3
3
  import { formatISODate, getTimeSeries } from '../../modules/utils';
4
4
  import { Activity } from './activity';
@@ -61,9 +61,9 @@ class Trigger extends Activity {
61
61
  return this.context.metadata.jid;
62
62
  } catch (error) {
63
63
  if (error instanceof DuplicateJobError) {
64
- this.logger.error('duplicate-job-error', error);
64
+ this.logger.error('duplicate-job-error', { error });
65
65
  } else {
66
- this.logger.error('trigger-process-error', error);
66
+ this.logger.error('trigger-process-error', { error });
67
67
  }
68
68
  telemetry.setActivityError(error.message);
69
69
  throw error;
@@ -140,6 +140,7 @@ class Trigger extends Activity {
140
140
  },
141
141
  };
142
142
  this.context['$self'] = this.context[this.metadata.aid];
143
+ this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
143
144
  }
144
145
 
145
146
  bindJobMetadataPaths(): string[] {
@@ -151,7 +152,7 @@ class Trigger extends Activity {
151
152
  }
152
153
 
153
154
  resolveGranularity(): string {
154
- return ReporterService.DEFAULT_GRANULARITY;
155
+ return this.config.stats?.granularity || ReporterService.DEFAULT_GRANULARITY;
155
156
  }
156
157
 
157
158
  getJobStatus(): number {
@@ -160,7 +161,7 @@ class Trigger extends Activity {
160
161
 
161
162
  resolveJobId(context: Partial<JobState>): string {
162
163
  const jobId = this.config.stats?.id;
163
- return jobId ? Pipe.resolve(jobId, context) : uuidv4();
164
+ return jobId ? Pipe.resolve(jobId, context) : nanoid();
164
165
  }
165
166
 
166
167
  resolveJobKey(context: Partial<JobState>): string {
@@ -8,13 +8,10 @@ import {
8
8
  ActivityType,
9
9
  WorkerActivity } from '../../types/activity';
10
10
  import { JobState } from '../../types/job';
11
- import { MultiResponseFlags } from '../../types/redis';
12
- import { StringScalarType } from '../../types/serializer';
13
- import {
14
- StreamCode,
15
- StreamData,
16
- StreamStatus } from '../../types/stream';
11
+ import { MultiResponseFlags, RedisMulti } from '../../types/redis';
12
+ import { StreamData} from '../../types/stream';
17
13
  import { TelemetryService } from '../telemetry';
14
+ import { Pipe } from '../pipe';
18
15
 
19
16
  class Worker extends Activity {
20
17
  config: WorkerActivity;
@@ -34,24 +31,26 @@ class Worker extends Activity {
34
31
  this.logger.debug('worker-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
35
32
  let telemetry: TelemetryService;
36
33
  try {
34
+ //confirm entry is allowed and restore state
37
35
  this.setLeg(1);
38
36
  await CollatorService.notarizeEntry(this);
39
-
40
37
  await this.getState();
41
38
  telemetry = new TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
42
39
  telemetry.startActivitySpan(this.leg);
43
40
  this.mapInputData();
44
41
 
42
+ //save state and authorize reentry
45
43
  const multi = this.store.getMulti();
46
44
  //todo: await this.registerTimeout();
45
+ const messageId = await this.execActivity(multi);
47
46
  await CollatorService.authorizeReentry(this, multi);
48
47
  await this.setState(multi);
49
48
  await this.setStatus(0, multi);
50
49
  const multiResponse = await multi.exec() as MultiResponseFlags;
51
50
 
51
+ //telemetry
52
52
  telemetry.mapActivityAttributes();
53
53
  const jobStatus = this.resolveStatus(multiResponse);
54
- const messageId = await this.execActivity();
55
54
  telemetry.setActivityAttributes({
56
55
  'app.activity.mid': messageId,
57
56
  'app.job.jss': jobStatus
@@ -60,9 +59,9 @@ class Worker extends Activity {
60
59
  return this.context.metadata.aid;
61
60
  } catch (error) {
62
61
  if (error instanceof GetStateError) {
63
- this.logger.error('worker-get-state-error', error);
62
+ this.logger.error('worker-get-state-error', { error });
64
63
  } else {
65
- this.logger.error('worker-process-error', error);
64
+ this.logger.error('worker-process-error', { error });
66
65
  }
67
66
  telemetry.setActivityError(error.message);
68
67
  throw error;
@@ -72,13 +71,14 @@ class Worker extends Activity {
72
71
  }
73
72
  }
74
73
 
75
- async execActivity(): Promise<string> {
74
+ async execActivity(multi: RedisMulti): Promise<string> {
75
+ const topic = Pipe.resolve(this.config.subtype, this.context);
76
76
  const streamData: StreamData = {
77
77
  metadata: {
78
78
  jid: this.context.metadata.jid,
79
79
  dad: this.metadata.dad,
80
80
  aid: this.metadata.aid,
81
- topic: this.config.subtype,
81
+ topic,
82
82
  spn: this.context['$self'].output.metadata.l1s,
83
83
  trc: this.context.metadata.trc,
84
84
  },
@@ -89,7 +89,7 @@ class Worker extends Activity {
89
89
  retry: this.config.retry
90
90
  };
91
91
  }
92
- return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData)) as string;
92
+ return (await this.engine.streamSignaler?.publishMessage(topic, streamData, multi)) as string;
93
93
  }
94
94
  }
95
95
 
@@ -8,6 +8,7 @@ import { HookRule } from '../../types/hook';
8
8
  import { HotMeshGraph, HotMeshManifest } from '../../types/hotmesh';
9
9
  import { RedisClient, RedisMulti } from '../../types/redis';
10
10
  import { StringAnyType, Symbols } from '../../types/serializer';
11
+ import { Pipe } from '../pipe';
11
12
 
12
13
  const DEFAULT_METADATA_RANGE_SIZE = 26; //metadata is 26 slots ([a-z] * 1)
13
14
  const DEFAULT_DATA_RANGE_SIZE = 260; //data is 260 slots ([a-zA-Z] * 5)
@@ -425,7 +426,8 @@ class Deployer {
425
426
  const activities = graph.activities;
426
427
  for (const activityKey in activities) {
427
428
  const activity = activities[activityKey];
428
- if (activity.type === 'worker') {
429
+ //only precreate if the topic is concrete and not `mappable`
430
+ if (activity.type === 'worker' && Pipe.resolve(activity.subtype, {}) === activity.subtype) {
429
431
  params.topic = activity.subtype;
430
432
  const key = this.store.mintKey(KeyType.STREAMS, params);
431
433
  //create one worker group per unique activity subtype (the topic)
@@ -1,4 +1,4 @@
1
- import { v4 as uuidv4 } from 'uuid';
1
+ import { nanoid } from 'nanoid';
2
2
 
3
3
  import { identifyRedisTypeFromClass } from '../../modules/utils';
4
4
  import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
@@ -23,14 +23,14 @@ export class ConnectorService {
23
23
  if (identifyRedisTypeFromClass(Redis) === 'redis') {
24
24
  for (let i = 1; i <= 3; i++) {
25
25
  instances.push(RedisConnection.connect(
26
- uuidv4(),
26
+ nanoid(),
27
27
  Redis as RedisClassType,
28
28
  options as RedisClientOptions));
29
29
  }
30
30
  } else {
31
31
  for (let i = 1; i <= 3; i++) {
32
32
  instances.push(IORedisConnection.connect(
33
- uuidv4(),
33
+ nanoid(),
34
34
  Redis as IORedisClassType,
35
35
  options as IORedisClientOptions));
36
36
  }
@@ -1,8 +1,14 @@
1
+ import { nanoid } from 'nanoid';
2
+ import { APP_ID, APP_VERSION, DEFAULT_COEFFICIENT, HOOK_ID, SUBSCRIBES_TOPIC, getWorkflowYAML } from './factory';
1
3
  import { WorkflowHandleService } from './handle';
2
4
  import { HotMeshService as HotMesh } from '../hotmesh';
3
- import { ClientConfig, Connection, WorkflowOptions } from '../../types/durable';
4
- import { getWorkflowYAML } from './factory';
5
+ import {
6
+ ClientConfig,
7
+ Connection,
8
+ SignalOptions,
9
+ WorkflowOptions } from '../../types/durable';
5
10
  import { JobState } from '../../types/job';
11
+ import { KeyType } from '../../modules/key';
6
12
 
7
13
  /*
8
14
  Here is an example of how the methods in this file are used:
@@ -12,7 +18,7 @@ Here is an example of how the methods in this file are used:
12
18
  import { Durable } from '@hotmeshio/hotmesh';
13
19
  import Redis from 'ioredis';
14
20
  import { example } from './workflows';
15
- import { v4 as uuidv4 } from 'uuid';
21
+ import { nanoid } from 'nanoid';
16
22
 
17
23
  async function run() {
18
24
  const connection = await Durable.Connection.connect({
@@ -31,7 +37,7 @@ async function run() {
31
37
  args: ['HotMesh'],
32
38
  taskQueue: 'hello-world',
33
39
  workflowName: 'example',
34
- workflowId: 'workflow-' + uuidv4(),
40
+ workflowId: 'workflow-' + nanoid(),
35
41
  });
36
42
 
37
43
  console.log(`Started workflow ${handle.workflowId}`);
@@ -55,13 +61,14 @@ export class ClientService {
55
61
  this.connection = config.connection;
56
62
  }
57
63
 
58
- getHotMesh = async (worflowTopic: string) => {
64
+ getHotMeshClient = async (worflowTopic: string) => {
65
+ //NOTE: every unique topic inits a new engine
59
66
  if (ClientService.instances.has(worflowTopic)) {
60
67
  return await ClientService.instances.get(worflowTopic);
61
68
  }
62
69
 
63
- const hotMesh = HotMesh.init({
64
- appId: worflowTopic,
70
+ const hotMeshClient = HotMesh.init({
71
+ appId: APP_ID,
65
72
  engine: {
66
73
  redis: {
67
74
  class: this.connection.class,
@@ -69,9 +76,19 @@ export class ClientService {
69
76
  }
70
77
  }
71
78
  });
72
- ClientService.instances.set(worflowTopic, hotMesh);
73
- await this.activateWorkflow(await hotMesh, worflowTopic);
74
- return hotMesh;
79
+ ClientService.instances.set(worflowTopic, hotMeshClient);
80
+
81
+ //since the YAML topic is dynamic, it MUST be manually created before use
82
+ const store = (await hotMeshClient).engine.store;
83
+ const params = { appId: APP_ID, topic: worflowTopic };
84
+ const streamKey = store.mintKey(KeyType.STREAMS, params);
85
+ try {
86
+ await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
87
+ } catch (err) {
88
+ //ignore if already exists
89
+ }
90
+ await this.activateWorkflow(await hotMeshClient);
91
+ return hotMeshClient;
75
92
  }
76
93
 
77
94
  workflow = {
@@ -80,36 +97,57 @@ export class ClientService {
80
97
  const workflowName = options.workflowName;
81
98
  const trc = options.workflowTrace;
82
99
  const spn = options.workflowSpan;
100
+ //topic is concat of taskQueue and workflowName
83
101
  const workflowTopic = `${taskQueueName}-${workflowName}`;
84
- const hotMesh = await this.getHotMesh(workflowTopic);
102
+ const hotMeshClient = await this.getHotMeshClient(workflowTopic);
85
103
  const payload = {
86
104
  arguments: [...options.args],
87
- workflowId: options.workflowId,
105
+ workflowId: options.workflowId || nanoid(),
106
+ workflowTopic: workflowTopic,
107
+ backoffCoefficient: options.config?.backoffCoefficient || DEFAULT_COEFFICIENT,
88
108
  }
89
109
  const context = { metadata: { trc, spn }, data: {}};
90
- const jobId = await hotMesh.pub(workflowTopic, payload, context as JobState);
91
- return new WorkflowHandleService(hotMesh, workflowTopic, jobId);
110
+ const jobId = await hotMeshClient.pub(
111
+ SUBSCRIBES_TOPIC,
112
+ payload,
113
+ context as JobState);
114
+ return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
92
115
  },
116
+
117
+ //signal in to activate a paused (waitForSignal) workflow
118
+ signal: async (options: SignalOptions): Promise<void> => {
119
+ const taskQueueName = options.taskQueue;
120
+ const workflowName = options.workflowName;
121
+ const workflowTopic = `${taskQueueName}-${workflowName}`;
122
+ const hotMeshClient = await this.getHotMeshClient(workflowTopic);
123
+ const payload = {
124
+ id: options.workflowId,
125
+ data: { ...options.data },
126
+ }
127
+ await hotMeshClient.hook(
128
+ HOOK_ID,
129
+ payload
130
+ );
131
+ }
93
132
  };
94
133
 
95
- async activateWorkflow(hotMesh: HotMesh, workflowTopic: string): Promise<void> {
96
- const version = '1';
97
- const app = await hotMesh.engine.store.getApp(workflowTopic);
134
+ async activateWorkflow(hotMesh: HotMesh, appId = APP_ID, version = APP_VERSION): Promise<void> {
135
+ const app = await hotMesh.engine.store.getApp(appId);
98
136
  const appVersion = app?.version as unknown as number;
99
137
  if(isNaN(appVersion)) {
100
138
  try {
101
- await hotMesh.deploy(getWorkflowYAML(workflowTopic, version));
139
+ await hotMesh.deploy(getWorkflowYAML(appId, version));
102
140
  await hotMesh.activate(version);
103
- } catch (err) {
104
- hotMesh.engine.logger.error('durable-client-deploy-activate-err', err);
105
- throw err;
141
+ } catch (error) {
142
+ hotMesh.engine.logger.error('durable-client-deploy-activate-err', { error });
143
+ throw error;
106
144
  }
107
145
  } else if(app && !app.active) {
108
146
  try {
109
147
  await hotMesh.activate(version);
110
- } catch (err) {
111
- hotMesh.engine.logger.error('durable-client-activate-err', err);
112
- throw err;
148
+ } catch (error) {
149
+ hotMesh.engine.logger.error('durable-client-activate-err', { error});
150
+ throw error;
113
151
  }
114
152
  }
115
153
  }
@@ -7,7 +7,7 @@ Here is an example of how the methods in this file are used:
7
7
 
8
8
  import { Durable } from '@hotmeshio/hotmesh';
9
9
  import Redis from 'ioredis';
10
- import { v4 as uuidv4 } from 'uuid';
10
+ import { nanoid } from 'nanoid';
11
11
 
12
12
  async function run() {
13
13
  const connection = await Durable.Connection.connect({
@@ -26,7 +26,7 @@ async function run() {
26
26
  taskQueue: 'hello-world',
27
27
  args: ['HotMesh'],
28
28
  workflowName: 'example',
29
- workflowId: uuidv4(),
29
+ workflowId: nanoid(),
30
30
  });
31
31
 
32
32
  console.log(`Started workflow ${handle.workflowId}`);