@hotmeshio/hotmesh 0.0.43 → 0.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/build/package.json +1 -1
  2. package/build/services/activities/trigger.js +7 -1
  3. package/build/services/durable/exporter.d.ts +105 -0
  4. package/build/services/durable/exporter.js +374 -0
  5. package/build/services/durable/factory.js +6 -63
  6. package/build/services/durable/handle.d.ts +4 -0
  7. package/build/services/durable/handle.js +5 -0
  8. package/build/services/durable/workflow.js +24 -21
  9. package/build/services/engine/index.d.ts +6 -1
  10. package/build/services/engine/index.js +9 -2
  11. package/build/services/exporter/index.d.ts +46 -0
  12. package/build/services/exporter/index.js +126 -0
  13. package/build/services/hotmesh/index.d.ts +4 -1
  14. package/build/services/hotmesh/index.js +6 -0
  15. package/build/services/quorum/index.js +2 -1
  16. package/build/services/router/index.d.ts +3 -0
  17. package/build/services/router/index.js +3 -0
  18. package/build/services/store/index.d.ts +5 -2
  19. package/build/services/store/index.js +54 -6
  20. package/build/services/task/index.js +5 -1
  21. package/build/services/worker/index.js +5 -4
  22. package/build/types/activity.d.ts +6 -1
  23. package/build/types/exporter.d.ts +51 -0
  24. package/build/types/exporter.js +8 -0
  25. package/build/types/index.d.ts +1 -0
  26. package/build/types/quorum.d.ts +1 -0
  27. package/build/types/task.d.ts +1 -1
  28. package/package.json +1 -1
  29. package/services/activities/trigger.ts +14 -0
  30. package/services/durable/exporter.ts +408 -0
  31. package/services/durable/factory.ts +6 -63
  32. package/services/durable/handle.ts +12 -0
  33. package/services/durable/workflow.ts +24 -22
  34. package/services/engine/index.ts +20 -5
  35. package/services/exporter/index.ts +147 -0
  36. package/services/hotmesh/index.ts +8 -1
  37. package/services/quorum/index.ts +2 -1
  38. package/services/router/index.ts +3 -0
  39. package/services/store/index.ts +56 -7
  40. package/services/task/index.ts +4 -1
  41. package/services/worker/index.ts +6 -5
  42. package/types/activity.ts +6 -1
  43. package/types/exporter.ts +61 -0
  44. package/types/index.ts +13 -1
  45. package/types/quorum.ts +1 -0
  46. package/types/task.ts +1 -1
@@ -37,7 +37,7 @@ class WorkflowService {
37
37
  const entityOrEmptyString = options.entity ?? '';
38
38
  //If the workflowId is not provided, it is generated from the entity and the workflow name
39
39
  const childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
40
- const parentWorkflowId = `${workflowId}-f`;
40
+ const parentWorkflowId = workflowId;
41
41
  const client = new client_1.ClientService({
42
42
  connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
43
43
  });
@@ -75,32 +75,35 @@ class WorkflowService {
75
75
  const workflowSpan = store.get('workflowSpan');
76
76
  const COUNTER = store.get('counter');
77
77
  const execIndex = COUNTER.counter = COUNTER.counter + 1;
78
+ const sessionId = `-start${workflowDimension}-${execIndex}-`;
78
79
  //NOTE: this is the hash prefix; necessary for the search index to locate the entity
79
80
  const entityOrEmptyString = options.entity ?? '';
80
81
  //If the workflowId is not provided, it is generated from the entity and the workflow name
81
- const childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
82
- const parentWorkflowId = `${workflowId}-f`;
82
+ const parentWorkflowId = workflowId;
83
83
  const workflowTopic = `${options.entity ?? options.taskQueue}-${options.entity ?? options.workflowName}`;
84
- try {
85
- //get the status; if there is no error, return childJobId (what was spawned)
86
- const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
87
- await hotMeshClient.getStatus(childJobId);
84
+ const hotMeshClient = await worker_1.WorkerService.getHotMesh(workflowTopic, { namespace });
85
+ const keyParams = { appId: hotMeshClient.appId, jobId: workflowId };
86
+ const workflowGuid = key_1.KeyService.mintKey(hotMeshClient.namespace, key_1.KeyType.JOB_STATE, keyParams);
87
+ let childJobId = await hotMeshClient.engine.store.exec('HGET', workflowGuid, sessionId);
88
+ if (childJobId) {
88
89
  return childJobId;
89
90
  }
90
- catch (error) {
91
- const client = new client_1.ClientService({
92
- connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
93
- });
94
- await client.workflow.start({
95
- ...options,
96
- namespace,
97
- workflowId: childJobId,
98
- parentWorkflowId,
99
- workflowTrace,
100
- workflowSpan,
101
- });
102
- return childJobId;
91
+ else {
92
+ childJobId = options.workflowId ?? `${entityOrEmptyString}-${workflowId}-$${options.entity ?? options.workflowName}${workflowDimension}-${execIndex}`;
103
93
  }
94
+ const client = new client_1.ClientService({
95
+ connection: await connection_1.ConnectionService.connect(worker_1.WorkerService.connection),
96
+ });
97
+ await client.workflow.start({
98
+ ...options,
99
+ namespace,
100
+ workflowId: childJobId,
101
+ parentWorkflowId,
102
+ workflowTrace,
103
+ workflowSpan,
104
+ });
105
+ await hotMeshClient.engine.store.exec('HSET', workflowGuid, sessionId, childJobId);
106
+ return childJobId;
104
107
  }
105
108
  /**
106
109
  * Wraps activities in a proxy that will durably run them
@@ -439,7 +442,7 @@ class WorkflowService {
439
442
  arguments: Array.from(arguments),
440
443
  //when the origin job is removed
441
444
  originJobId: originJobId ?? workflowId,
442
- parentWorkflowId: `${workflowId}-a`,
445
+ parentWorkflowId: workflowId,
443
446
  workflowId: activityJobId,
444
447
  workflowTopic: activityTopic,
445
448
  activityName,
@@ -5,6 +5,7 @@ import { Interrupt } from '../activities/interrupt';
5
5
  import { Signal } from '../activities/signal';
6
6
  import { Worker } from '../activities/worker';
7
7
  import { Trigger } from '../activities/trigger';
8
+ import { ExporterService } from '../exporter';
8
9
  import { ILogger } from '../logger';
9
10
  import { Router } from '../router';
10
11
  import { StoreService } from '../store';
@@ -18,15 +19,17 @@ import { JobState, JobData, JobMetadata, JobOutput, JobStatus, JobInterruptOptio
18
19
  import { HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshSettings } from '../../types/hotmesh';
19
20
  import { JobMessageCallback } from '../../types/quorum';
20
21
  import { RedisClient, RedisMulti } from '../../types/redis';
21
- import { StringAnyType } from '../../types/serializer';
22
+ import { StringAnyType, StringStringType } from '../../types/serializer';
22
23
  import { GetStatsOptions, IdsResponse, JobStatsInput, StatsResponse } from '../../types/stats';
23
24
  import { StreamCode, StreamData, StreamDataResponse, StreamError, StreamStatus } from '../../types/stream';
24
25
  import { WorkListTaskType } from '../../types/task';
26
+ import { JobExport } from '../../types/exporter';
25
27
  declare class EngineService {
26
28
  namespace: string;
27
29
  apps: HotMeshApps | null;
28
30
  appId: string;
29
31
  guid: string;
32
+ exporter: ExporterService | null;
30
33
  router: Router | null;
31
34
  store: StoreService<RedisClient, RedisMulti> | null;
32
35
  stream: StreamService<RedisClient, RedisMulti> | null;
@@ -88,6 +91,8 @@ declare class EngineService {
88
91
  * it will be expired immediately.
89
92
  */
90
93
  resolveExpires(context: JobState, options: JobCompletionOptions): number;
94
+ export(jobId: string): Promise<JobExport>;
95
+ getRaw(jobId: string): Promise<StringStringType>;
91
96
  getStatus(jobId: string): Promise<JobStatus>;
92
97
  getState(topic: string, jobId: string): Promise<JobOutput>;
93
98
  getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
@@ -9,6 +9,7 @@ const enums_1 = require("../../modules/enums");
9
9
  const utils_1 = require("../../modules/utils");
10
10
  const activities_1 = __importDefault(require("../activities"));
11
11
  const compiler_1 = require("../compiler");
12
+ const exporter_1 = require("../exporter");
12
13
  const reporter_1 = require("../reporter");
13
14
  const router_1 = require("../router");
14
15
  const serializer_1 = require("../serializer");
@@ -41,8 +42,8 @@ class EngineService {
41
42
  await instance.initStreamChannel(config.engine.stream);
42
43
  instance.router = instance.initRouter(config);
43
44
  instance.router.consumeMessages(instance.stream.mintKey(key_1.KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
44
- //the task service is used by the engine to process `webhooks` and `timehooks`
45
45
  instance.taskService = new task_1.TaskService(instance.store, logger);
46
+ instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
46
47
  return instance;
47
48
  }
48
49
  }
@@ -526,9 +527,15 @@ class EngineService {
526
527
  return options.expire ?? context.metadata.expire ?? enums_1.HMSH_EXPIRE_JOB_SECONDS;
527
528
  }
528
529
  // ****** GET JOB STATE/COLLATION STATUS BY ID *********
530
+ async export(jobId) {
531
+ return await this.exporter.export(jobId);
532
+ }
533
+ async getRaw(jobId) {
534
+ return await this.store.getRaw(jobId);
535
+ }
529
536
  async getStatus(jobId) {
530
537
  const { id: appId } = await this.getVID();
531
- return this.store.getStatus(jobId, appId);
538
+ return await this.store.getStatus(jobId, appId);
532
539
  }
533
540
  //todo: add 'options' parameter;
534
541
  // (e.g, if {dimensions:true}, use hscan to deliver
@@ -0,0 +1,46 @@
1
+ import { ILogger } from '../logger';
2
+ import { StoreService } from '../store';
3
+ import { StringStringType, Symbols } from "../../types/serializer";
4
+ import { RedisClient, RedisMulti } from '../../types/redis';
5
+ import { DependencyExport, ExportOptions, JobActionExport, JobExport } from '../../types/exporter';
6
+ import { SerializerService } from '../serializer';
7
+ /**
8
+ * Downloads job data from Redis (hscan, hmget, hgetall)
9
+ * Expands process data and includes dependency list
10
+ */
11
+ declare class ExporterService {
12
+ appId: string;
13
+ logger: ILogger;
14
+ serializer: SerializerService;
15
+ store: StoreService<RedisClient, RedisMulti>;
16
+ symbols: Promise<Symbols> | Symbols;
17
+ constructor(appId: string, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
18
+ /**
19
+ * Convert the job hash and dependency list into a JobExport object.
20
+ * This object contains various facets that describe the interaction
21
+ * in terms relevant to narrative storytelling.
22
+ */
23
+ export(jobId: string, options?: ExportOptions): Promise<JobExport>;
24
+ /**
25
+ * Inflates the key from Redis, 3-character symbol
26
+ * into a human-readable JSON path, reflecting the
27
+ * tree-like structure of the unidimensional Hash
28
+ */
29
+ inflateKey(key: string): string;
30
+ /**
31
+ * Inflates the job data from Redis into a JobExport object
32
+ * @param jobHash - the job data from Redis
33
+ * @param dependencyList - the list of dependencies for the job
34
+ * @returns - the inflated job data
35
+ */
36
+ inflate(jobHash: StringStringType, dependencyList: string[]): JobExport;
37
+ /**
38
+ * Inflates the dependency data from Redis into a JobExport object by
39
+ * organizing the dimensional isolate in sch a way asto interleave
40
+ * into a story
41
+ * @param data - the dependency data from Redis
42
+ * @returns - the organized dependency data
43
+ */
44
+ inflateDependencyData(data: string[], actions: JobActionExport): DependencyExport[];
45
+ }
46
+ export { ExporterService };
@@ -0,0 +1,126 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ExporterService = void 0;
4
+ const serializer_1 = require("../serializer");
5
+ const utils_1 = require("../../modules/utils");
6
+ /**
7
+ * Downloads job data from Redis (hscan, hmget, hgetall)
8
+ * Expands process data and includes dependency list
9
+ */
10
+ class ExporterService {
11
+ constructor(appId, store, logger) {
12
+ this.appId = appId;
13
+ this.logger = logger;
14
+ this.store = store;
15
+ this.serializer = new serializer_1.SerializerService();
16
+ }
17
+ /**
18
+ * Convert the job hash and dependency list into a JobExport object.
19
+ * This object contains various facets that describe the interaction
20
+ * in terms relevant to narrative storytelling.
21
+ */
22
+ async export(jobId, options = {}) {
23
+ if (!this.symbols) {
24
+ this.symbols = this.store.getAllSymbols();
25
+ this.symbols = await this.symbols;
26
+ }
27
+ const depData = await this.store.getDependencies(jobId);
28
+ const jobData = await this.store.getRaw(jobId);
29
+ const jobExport = this.inflate(jobData, depData);
30
+ return jobExport;
31
+ }
32
+ /**
33
+ * Inflates the key from Redis, 3-character symbol
34
+ * into a human-readable JSON path, reflecting the
35
+ * tree-like structure of the unidimensional Hash
36
+ */
37
+ inflateKey(key) {
38
+ return (key in this.symbols) ? this.symbols[key] : key;
39
+ }
40
+ /**
41
+ * Inflates the job data from Redis into a JobExport object
42
+ * @param jobHash - the job data from Redis
43
+ * @param dependencyList - the list of dependencies for the job
44
+ * @returns - the inflated job data
45
+ */
46
+ inflate(jobHash, dependencyList) {
47
+ //the list of actions taken in the workflow and hook functions
48
+ const actions = {
49
+ hooks: {},
50
+ main: {
51
+ cursor: -1,
52
+ items: []
53
+ }
54
+ };
55
+ const process = {};
56
+ const dependencies = this.inflateDependencyData(dependencyList, actions);
57
+ const regex = /^([a-zA-Z]{3}),(\d+(?:,\d+)*)/;
58
+ Object.entries(jobHash).forEach(([key, value]) => {
59
+ const match = key.match(regex);
60
+ if (match) {
61
+ //activity process state
62
+ const [_, letters, numbers] = match;
63
+ const path = this.inflateKey(letters);
64
+ const dimensions = `${numbers.replace(/,/g, '/')}`;
65
+ const resolved = this.serializer.fromString(value);
66
+ process[`${dimensions}/${path}`] = resolved;
67
+ }
68
+ else if (key.length === 3) {
69
+ //job state
70
+ process[this.inflateKey(key)] = this.serializer.fromString(value);
71
+ }
72
+ });
73
+ return {
74
+ dependencies,
75
+ process: (0, utils_1.restoreHierarchy)(process),
76
+ status: jobHash[':'],
77
+ };
78
+ }
79
+ /**
80
+ * Inflates the dependency data from Redis into a JobExport object by
81
+ * organizing the dimensional isolate in sch a way asto interleave
82
+ * into a story
83
+ * @param data - the dependency data from Redis
84
+ * @returns - the organized dependency data
85
+ */
86
+ inflateDependencyData(data, actions) {
87
+ const hookReg = /([0-9,]+)-(\d+)$/;
88
+ const flowReg = /-(\d+)$/;
89
+ return data.map((dependency, index) => {
90
+ const [action, topic, gid, ...jid] = dependency.split('::');
91
+ const jobId = jid.join('::');
92
+ const match = jobId.match(hookReg);
93
+ let prefix;
94
+ let type;
95
+ let dimensionKey = '';
96
+ if (match) {
97
+ //hook-originating dependency
98
+ const [_, dimension, counter] = match;
99
+ dimensionKey = dimension.split(',').join('/');
100
+ prefix = `${dimensionKey}[${counter}]`;
101
+ type = 'hook';
102
+ }
103
+ else {
104
+ const match = jobId.match(flowReg);
105
+ if (match) {
106
+ //main workflow-originating dependency
107
+ const [_, counter] = match;
108
+ prefix = `[${counter}]`;
109
+ type = 'flow';
110
+ }
111
+ else {
112
+ //'other' types like signal cleanup
113
+ prefix = '/';
114
+ type = 'other';
115
+ }
116
+ }
117
+ return {
118
+ type: action,
119
+ topic,
120
+ gid,
121
+ jid: jobId,
122
+ };
123
+ });
124
+ }
125
+ }
126
+ exports.ExporterService = ExporterService;
@@ -7,7 +7,8 @@ import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
7
7
  import { JobMessageCallback, QuorumProfile } from '../../types/quorum';
8
8
  import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
9
9
  import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
10
- import { StringAnyType } from '../../types/serializer';
10
+ import { StringAnyType, StringStringType } from '../../types/serializer';
11
+ import { JobExport } from '../../types/exporter';
11
12
  declare class HotMeshService {
12
13
  namespace: string;
13
14
  appId: string;
@@ -35,6 +36,8 @@ declare class HotMeshService {
35
36
  plan(path: string): Promise<HotMeshManifest>;
36
37
  deploy(pathOrYAML: string): Promise<HotMeshManifest>;
37
38
  activate(version: string, delay?: number): Promise<boolean>;
39
+ export(jobId: string): Promise<JobExport>;
40
+ getRaw(jobId: string): Promise<StringStringType>;
38
41
  getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
39
42
  getStatus(jobId: string): Promise<JobStatus>;
40
43
  getState(topic: string, jobId: string): Promise<JobOutput>;
@@ -104,6 +104,12 @@ class HotMeshService {
104
104
  return await this.quorum?.activate(version, delay);
105
105
  }
106
106
  // ************* REPORTER METHODS *************
107
+ async export(jobId) {
108
+ return await this.engine?.export(jobId);
109
+ }
110
+ async getRaw(jobId) {
111
+ return await this.engine?.getRaw(jobId);
112
+ }
107
113
  async getStats(topic, query) {
108
114
  return await this.engine?.getStats(topic, query);
109
115
  }
@@ -103,7 +103,8 @@ class QuorumService {
103
103
  engine_id: this.guid,
104
104
  namespace: this.namespace,
105
105
  app_id: this.appId,
106
- stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId })
106
+ stream: this.engine.stream.mintKey(hotmesh_1.KeyType.STREAMS, { appId: this.appId }),
107
+ counts: this.engine.router.counts,
107
108
  };
108
109
  }
109
110
  this.store.publish(hotmesh_1.KeyType.QUORUM, {
@@ -17,6 +17,9 @@ declare class Router {
17
17
  logger: ILogger;
18
18
  throttle: number;
19
19
  errorCount: number;
20
+ counts: {
21
+ [key: string]: number;
22
+ };
20
23
  currentTimerId: NodeJS.Timeout | null;
21
24
  shouldConsume: boolean;
22
25
  constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
@@ -10,6 +10,7 @@ class Router {
10
10
  constructor(config, stream, store, logger) {
11
11
  this.throttle = 0;
12
12
  this.errorCount = 0;
13
+ this.counts = {};
13
14
  this.currentTimerId = null;
14
15
  this.appId = config.appId;
15
16
  this.guid = config.guid;
@@ -146,6 +147,8 @@ class Router {
146
147
  else {
147
148
  output.metadata.guid = (0, utils_1.guid)();
148
149
  }
150
+ const code = output.code || 200;
151
+ this.counts[code] = (this.counts[code] || 0) + 1;
149
152
  output.type = stream_1.StreamDataType.RESPONSE;
150
153
  return await this.publishMessage(null, output);
151
154
  }
@@ -51,6 +51,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
51
51
  getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
52
52
  setSettings(manifest: HotMeshSettings): Promise<any>;
53
53
  reserveSymbolRange(target: string, size: number, type: 'JOB' | 'ACTIVITY'): Promise<[number, number, Symbols]>;
54
+ getAllSymbols(): Promise<Symbols>;
54
55
  getSymbols(activityId: string): Promise<Symbols>;
55
56
  addSymbols(activityId: string, symbols: Symbols): Promise<boolean>;
56
57
  seedSymbols(target: string, type: 'JOB' | 'ACTIVITY', startIndex: number): StringStringType;
@@ -68,7 +69,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
68
69
  * when `originJobId` is interrupted/expired, the items in the
69
70
  * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
70
71
  */
71
- registerJobDependency(originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
72
+ registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi?: U): Promise<any>;
72
73
  /**
73
74
  * Ensures a `hook signal` is delisted when its parent activity/job
74
75
  * is interrupted/expired.
@@ -87,6 +88,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
87
88
  */
88
89
  getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
89
90
  getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
91
+ getRaw(jobId: string): Promise<StringStringType>;
90
92
  /**
91
93
  * collate is a generic method for incrementing a value in a hash
92
94
  * in order to track their progress during processing.
@@ -124,6 +126,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
124
126
  * is a standard `expire` or an `interrupt`
125
127
  */
126
128
  registerDependenciesForCleanup(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void>;
129
+ getDependencies(jobId: string): Promise<string[]>;
127
130
  /**
128
131
  * registers a hook activity to be awakened (uses ZSET to
129
132
  * store the 'sleep group' and LIST to store the events
@@ -140,7 +143,7 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
140
143
  * generic LIST (lists typically contain target job ids)
141
144
  * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
142
145
  */
143
- resolveTaskKeyContext(listKey: string): [('sleep' | 'expire' | 'interrupt' | 'delist'), string];
146
+ resolveTaskKeyContext(listKey: string): [WorkListTaskType, string];
144
147
  /**
145
148
  * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
146
149
  * This method is called by the engine and not by an activity and is
@@ -164,6 +164,35 @@ class StoreService {
164
164
  return [actualLowerLimit, upperLimit, symbols];
165
165
  }
166
166
  }
167
+ async getAllSymbols() {
168
+ //get hash with all reserved symbol ranges
169
+ const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
170
+ const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
171
+ const rangeKeys = Object.keys(ranges).sort();
172
+ delete rangeKeys[':cursor'];
173
+ const multi = this.getMulti();
174
+ for (const rangeKey of rangeKeys) {
175
+ const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, { activityId: rangeKey, appId: this.appId });
176
+ multi[this.commands.hgetall](symbolKey);
177
+ }
178
+ const results = await multi.exec();
179
+ const symbolSets = {};
180
+ results.forEach((result, index) => {
181
+ if (result) {
182
+ let vals;
183
+ if (Array.isArray(result) && result.length === 2) {
184
+ vals = result[1];
185
+ }
186
+ else {
187
+ vals = result;
188
+ }
189
+ for (const [key, value] of Object.entries(vals)) {
190
+ symbolSets[value] = key.startsWith(rangeKeys[index]) ? key : `${rangeKeys[index]}/${key}`;
191
+ }
192
+ }
193
+ });
194
+ return symbolSets;
195
+ }
167
196
  async getSymbols(activityId) {
168
197
  let symbols = this.cache.getSymbols(this.appId, activityId);
169
198
  if (symbols) {
@@ -314,15 +343,15 @@ class StoreService {
314
343
  * when `originJobId` is interrupted/expired, the items in the
315
344
  * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
316
345
  */
317
- async registerJobDependency(originJobId, topic, jobId, gId, multi) {
346
+ async registerJobDependency(depType, originJobId, topic, jobId, gId, multi) {
318
347
  const privateMulti = multi || this.getMulti();
319
348
  const dependencyParams = {
320
349
  appId: this.appId,
321
350
  jobId: originJobId,
322
351
  };
323
352
  const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, dependencyParams);
324
- //tasks have '4' segments
325
- const expireTask = `expire::${topic}::${gId}::${jobId}`;
353
+ //items listed as job dependencies have different relationships
354
+ const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
326
355
  privateMulti[this.commands.rpush](depKey, expireTask);
327
356
  if (!multi) {
328
357
  return await privateMulti.exec();
@@ -487,6 +516,14 @@ class StoreService {
487
516
  throw new errors_1.GetStateError(jobId);
488
517
  }
489
518
  }
519
+ async getRaw(jobId) {
520
+ const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
521
+ const job = await this.redisClient[this.commands.hgetall](jobKey);
522
+ if (!job) {
523
+ throw new errors_1.GetStateError(jobId);
524
+ }
525
+ return job;
526
+ }
490
527
  /**
491
528
  * collate is a generic method for incrementing a value in a hash
492
529
  * in order to track their progress during processing.
@@ -720,6 +757,11 @@ class StoreService {
720
757
  const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
721
758
  await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
722
759
  }
760
+ async getDependencies(jobId) {
761
+ const depParams = { appId: this.appId, jobId };
762
+ const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
763
+ return this.redisClient[this.commands.lrange](depKey, 0, -1);
764
+ }
723
765
  /**
724
766
  * registers a hook activity to be awakened (uses ZSET to
725
767
  * store the 'sleep group' and LIST to store the events
@@ -742,12 +784,18 @@ class StoreService {
742
784
  let [pType, pKey] = this.resolveTaskKeyContext(listKey);
743
785
  const timeEvent = await this.redisClient[this.commands.lpop](pKey);
744
786
  if (timeEvent) {
745
- //there are 4 time-related task
746
- //1) sleep (awaken), 2) expire, 3) interrupt, 4) delist
747
- const [type, activityId, gId, ...jobId] = timeEvent.split('::');
787
+ //there are task types
788
+ //1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
789
+ let [type, activityId, gId, ...jobId] = timeEvent.split('::');
748
790
  if (type === 'delist') {
749
791
  pType = 'delist';
750
792
  }
793
+ else if (type === 'child') {
794
+ pType = 'child';
795
+ }
796
+ else if (type === 'expire-child') {
797
+ type = 'expire'; //use the same logic as 'expire'
798
+ }
751
799
  return [listKey, jobId.join('::'), gId, activityId, pType];
752
800
  }
753
801
  await this.redisClient[this.commands.zrem](zsetKey, listKey);
@@ -75,7 +75,11 @@ class TaskService {
75
75
  const workListTask = await this.store.getNextTask(listKey);
76
76
  if (Array.isArray(workListTask)) {
77
77
  const [listKey, target, gId, activityId, type] = workListTask;
78
- if (type === 'delist') {
78
+ if (type === 'child') {
79
+ //continue; this child is listed here for convenience, but
80
+ // will be expired by an origin ancestor and is listed there
81
+ }
82
+ else if (type === 'delist') {
79
83
  //delist the signalKey (target)
80
84
  const key = this.store.mintKey(hotmesh_1.KeyType.SIGNALS, { appId: this.store.appId });
81
85
  await this.store.redisClient[this.store.commands.hdel](key, target);
@@ -2,16 +2,16 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.WorkerService = void 0;
4
4
  const key_1 = require("../../modules/key");
5
+ const utils_1 = require("../../modules/utils");
6
+ const connector_1 = require("../connector");
5
7
  const router_1 = require("../router");
6
- const redis_1 = require("../store/clients/redis");
7
8
  const ioredis_1 = require("../store/clients/ioredis");
8
- const redis_2 = require("../stream/clients/redis");
9
+ const redis_1 = require("../store/clients/redis");
9
10
  const ioredis_2 = require("../stream/clients/ioredis");
11
+ const redis_2 = require("../stream/clients/redis");
10
12
  const ioredis_3 = require("../sub/clients/ioredis");
11
13
  const redis_3 = require("../sub/clients/redis");
12
14
  const stream_1 = require("../../types/stream");
13
- const utils_1 = require("../../modules/utils");
14
- const connector_1 = require("../connector");
15
15
  class WorkerService {
16
16
  constructor() {
17
17
  this.reporting = false;
@@ -114,6 +114,7 @@ class WorkerService {
114
114
  app_id: this.appId,
115
115
  worker_topic: this.topic,
116
116
  stream: this.stream.mintKey(key_1.KeyType.STREAMS, params),
117
+ counts: this.router.counts,
117
118
  };
118
119
  }
119
120
  this.store.publish(key_1.KeyType.QUORUM, {
@@ -33,13 +33,18 @@ interface Measure {
33
33
  }
34
34
  interface TriggerActivityStats {
35
35
  /**
36
- * parent job; including this allows the parent's
36
+ * dependent parent job id; including this allows the parent's
37
37
  * expiration/interruption events to cascade; set
38
38
  * `expire` in the YAML for the dependent graph
39
39
  * to 0 and provide the parent for dependent,
40
40
  * cascading interruption and cleanup
41
41
  */
42
42
  parent?: string;
43
+ /**
44
+ * adjacent parent job id; this is the actual adjacent
45
+ * parent in the graph, but it is not used for cascading expiration
46
+ */
47
+ adjacent?: string;
43
48
  id?: {
44
49
  [key: string]: unknown;
45
50
  } | string;
@@ -0,0 +1,51 @@
1
+ import { StringAnyType } from "./serializer";
2
+ export type ExportItem = [(string | null), string, any];
3
+ export interface ExportOptions {
4
+ }
5
+ export type JobAction = {
6
+ cursor: number;
7
+ items: ExportItem[];
8
+ };
9
+ export interface JobActionExport {
10
+ hooks: {
11
+ [key: string]: JobAction;
12
+ };
13
+ main: JobAction;
14
+ }
15
+ export interface ActivityAction {
16
+ action: string;
17
+ target: string;
18
+ }
19
+ export interface JobTimeline {
20
+ activity: string;
21
+ dimension: string;
22
+ duplex: 'entry' | 'exit';
23
+ timestamp: string;
24
+ actions?: ActivityAction[];
25
+ }
26
+ export interface DependencyExport {
27
+ type: string;
28
+ topic: string;
29
+ gid: string;
30
+ jid: string;
31
+ }
32
+ export interface ExportTransitions {
33
+ [key: string]: string[];
34
+ }
35
+ export interface ExportCycles {
36
+ [key: string]: string[];
37
+ }
38
+ export interface DurableJobExport {
39
+ data: StringAnyType;
40
+ dependencies: DependencyExport[];
41
+ state: StringAnyType;
42
+ status: string;
43
+ timeline: JobTimeline[];
44
+ transitions: ExportTransitions;
45
+ cycles: ExportCycles;
46
+ }
47
+ export interface JobExport {
48
+ dependencies: DependencyExport[];
49
+ process: StringAnyType;
50
+ status: string;
51
+ }
@@ -0,0 +1,8 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ ;
4
+ ;
5
+ ;
6
+ ;
7
+ ;
8
+ ;