@hotmeshio/hotmesh 0.3.6 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/build/index.d.ts +2 -1
  2. package/build/index.js +7 -1
  3. package/build/modules/key.js +1 -62
  4. package/build/modules/utils.js +1 -267
  5. package/build/package.json +12 -8
  6. package/build/services/activities/activity.js +1 -495
  7. package/build/services/activities/await.js +1 -109
  8. package/build/services/activities/cycle.js +1 -96
  9. package/build/services/activities/hook.js +1 -154
  10. package/build/services/activities/index.js +1 -20
  11. package/build/services/activities/interrupt.js +1 -149
  12. package/build/services/activities/signal.js +1 -118
  13. package/build/services/activities/trigger.js +1 -237
  14. package/build/services/activities/worker.js +1 -101
  15. package/build/services/collator/index.js +1 -197
  16. package/build/services/compiler/deployer.d.ts +3 -1
  17. package/build/services/compiler/deployer.js +1 -455
  18. package/build/services/compiler/index.d.ts +3 -1
  19. package/build/services/compiler/index.js +1 -91
  20. package/build/services/compiler/validator.js +1 -122
  21. package/build/services/engine/index.d.ts +5 -2
  22. package/build/services/engine/index.js +1 -562
  23. package/build/services/exporter/index.js +1 -93
  24. package/build/services/mapper/index.js +1 -67
  25. package/build/services/meshdata/index.d.ts +0 -1
  26. package/build/services/meshdata/index.js +16 -24
  27. package/build/services/meshflow/client.js +4 -8
  28. package/build/services/meshflow/exporter.js +1 -186
  29. package/build/services/meshflow/search.d.ts +4 -5
  30. package/build/services/meshflow/search.js +45 -35
  31. package/build/services/meshflow/workflow.d.ts +1 -1
  32. package/build/services/meshflow/workflow.js +3 -28
  33. package/build/services/pipe/functions/array.js +1 -74
  34. package/build/services/pipe/functions/bitwise.js +1 -24
  35. package/build/services/pipe/functions/conditional.js +1 -36
  36. package/build/services/pipe/functions/cron.js +1 -32
  37. package/build/services/pipe/functions/date.js +1 -164
  38. package/build/services/pipe/functions/index.js +1 -30
  39. package/build/services/pipe/functions/json.js +1 -12
  40. package/build/services/pipe/functions/logical.js +1 -12
  41. package/build/services/pipe/functions/math.js +1 -182
  42. package/build/services/pipe/functions/number.js +1 -60
  43. package/build/services/pipe/functions/object.js +1 -81
  44. package/build/services/pipe/functions/string.js +1 -69
  45. package/build/services/pipe/functions/symbol.js +1 -33
  46. package/build/services/pipe/functions/unary.js +1 -18
  47. package/build/services/pipe/index.js +1 -221
  48. package/build/services/quorum/index.d.ts +1 -1
  49. package/build/services/quorum/index.js +1 -219
  50. package/build/services/reporter/index.js +1 -331
  51. package/build/services/router/index.js +1 -420
  52. package/build/services/search/factory.d.ts +7 -0
  53. package/build/services/search/factory.js +20 -0
  54. package/build/services/search/index.d.ts +21 -0
  55. package/build/services/search/index.js +10 -0
  56. package/build/services/search/providers/redis/ioredis.d.ts +18 -0
  57. package/build/services/search/providers/redis/ioredis.js +1 -0
  58. package/build/services/search/providers/redis/redis.d.ts +18 -0
  59. package/build/services/search/providers/redis/redis.js +1 -0
  60. package/build/services/serializer/index.js +1 -265
  61. package/build/services/store/factory.d.ts +2 -1
  62. package/build/services/store/factory.js +2 -2
  63. package/build/services/store/index.d.ts +71 -97
  64. package/build/services/store/index.js +2 -939
  65. package/build/services/store/providers/postgres/postgres.d.ts +0 -0
  66. package/build/services/store/providers/postgres/postgres.js +0 -0
  67. package/build/services/store/providers/postgres/types/hash.d.ts +0 -0
  68. package/build/services/store/providers/postgres/types/hash.js +0 -0
  69. package/build/services/store/providers/postgres/types/list.d.ts +0 -0
  70. package/build/services/store/providers/postgres/types/list.js +0 -0
  71. package/build/services/store/providers/postgres/types/string.d.ts +0 -0
  72. package/build/services/store/providers/postgres/types/string.js +0 -0
  73. package/build/services/store/providers/postgres/types/zset.d.ts +0 -0
  74. package/build/services/store/providers/postgres/types/zset.js +0 -0
  75. package/build/services/store/providers/redis/_base.d.ts +98 -0
  76. package/build/services/store/providers/redis/_base.js +1 -0
  77. package/build/services/store/providers/redis/ioredis.d.ts +12 -0
  78. package/build/services/store/providers/redis/ioredis.js +1 -0
  79. package/build/services/store/providers/redis/redis.d.ts +13 -0
  80. package/build/services/store/providers/redis/redis.js +1 -0
  81. package/build/services/store/providers/store-initializable.d.ts +5 -0
  82. package/build/services/store/providers/store-initializable.js +1 -0
  83. package/build/services/stream/factory.d.ts +2 -1
  84. package/build/services/stream/factory.js +5 -5
  85. package/build/services/stream/index.d.ts +13 -14
  86. package/build/services/stream/index.js +3 -2
  87. package/build/services/stream/providers/postgres/_deploy.d.ts +4 -0
  88. package/build/services/stream/providers/postgres/_deploy.js +1 -0
  89. package/build/services/stream/providers/redis/ioredis.d.ts +21 -0
  90. package/build/services/stream/providers/redis/ioredis.js +1 -0
  91. package/build/services/stream/providers/redis/redis.d.ts +21 -0
  92. package/build/services/stream/providers/redis/redis.js +1 -0
  93. package/build/services/stream/providers/stream-initializable.d.ts +5 -0
  94. package/build/services/stream/providers/stream-initializable.js +1 -0
  95. package/build/services/sub/factory.d.ts +1 -1
  96. package/build/services/sub/factory.js +5 -5
  97. package/build/services/sub/index.d.ts +9 -7
  98. package/build/services/sub/index.js +3 -2
  99. package/build/services/sub/{clients → providers/redis}/ioredis.d.ts +7 -10
  100. package/build/services/sub/providers/redis/ioredis.js +1 -0
  101. package/build/services/sub/{clients → providers/redis}/redis.d.ts +7 -10
  102. package/build/services/sub/providers/redis/redis.js +1 -0
  103. package/build/services/task/index.js +1 -171
  104. package/build/services/telemetry/index.js +1 -225
  105. package/build/services/worker/index.d.ts +2 -2
  106. package/build/services/worker/index.js +1 -158
  107. package/build/types/redis.d.ts +5 -5
  108. package/index.ts +15 -1
  109. package/package.json +12 -8
  110. package/types/redis.ts +5 -5
  111. package/build/services/store/clients/ioredis.d.ts +0 -30
  112. package/build/services/store/clients/ioredis.js +0 -220
  113. package/build/services/store/clients/redis.d.ts +0 -32
  114. package/build/services/store/clients/redis.js +0 -319
  115. package/build/services/stream/clients/ioredis.d.ts +0 -24
  116. package/build/services/stream/clients/ioredis.js +0 -121
  117. package/build/services/stream/clients/redis.d.ts +0 -24
  118. package/build/services/stream/clients/redis.js +0 -161
  119. package/build/services/sub/clients/ioredis.js +0 -72
  120. package/build/services/sub/clients/redis.js +0 -63
@@ -1,946 +1,9 @@
1
1
  "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
2
  Object.defineProperty(exports, "__esModule", { value: true });
26
3
  exports.StoreService = void 0;
27
- const errors_1 = require("../../modules/errors");
28
- const key_1 = require("../../modules/key");
29
- const serializer_1 = require("../serializer");
30
- const utils_1 = require("../../modules/utils");
31
- const enums_1 = require("../../modules/enums");
32
- const cache_1 = require("./cache");
33
4
  class StoreService {
34
- constructor(redisClient) {
35
- this.commands = {
36
- get: 'get',
37
- set: 'set',
38
- setnx: 'setnx',
39
- del: 'del',
40
- expire: 'expire',
41
- hset: 'hset',
42
- hscan: 'hscan',
43
- hsetnx: 'hsetnx',
44
- hincrby: 'hincrby',
45
- hdel: 'hdel',
46
- hget: 'hget',
47
- hmget: 'hmget',
48
- hgetall: 'hgetall',
49
- hincrbyfloat: 'hincrbyfloat',
50
- zrange: 'zrange',
51
- zrangebyscore_withscores: 'zrangebyscore',
52
- zrangebyscore: 'zrangebyscore',
53
- zrem: 'zrem',
54
- zadd: 'zadd',
55
- lmove: 'lmove',
56
- llen: 'llen',
57
- lpop: 'lpop',
58
- lrange: 'lrange',
59
- rename: 'rename',
60
- rpush: 'rpush',
61
- scan: 'scan',
62
- xack: 'xack',
63
- xdel: 'xdel',
64
- };
65
- this.redisClient = redisClient;
66
- }
67
- async init(namespace = key_1.HMNS, appId, logger) {
68
- this.namespace = namespace;
69
- this.appId = appId;
70
- this.logger = logger;
71
- const settings = await this.getSettings(true);
72
- this.cache = new cache_1.Cache(appId, settings);
73
- this.serializer = new serializer_1.SerializerService();
74
- await this.getApp(appId);
75
- return this.cache.getApps();
76
- }
77
- isSuccessful(result) {
78
- return result > 0 || result === 'OK' || result === true;
79
- }
80
- async zAdd(key, score, value, redisMulti) {
81
- return await (redisMulti || this.redisClient)[this.commands.zadd](key, score, value);
82
- }
83
- async zRangeByScoreWithScores(key, score, value) {
84
- const result = await this.redisClient[this.commands.zrangebyscore_withscores](key, score, value, 'WITHSCORES');
85
- if (result?.length > 0) {
86
- return result[0];
87
- }
88
- return null;
89
- }
90
- async zRangeByScore(key, score, value) {
91
- const result = await this.redisClient[this.commands.zrangebyscore](key, score, value);
92
- if (result?.length > 0) {
93
- return result[0];
94
- }
95
- return null;
96
- }
97
- mintKey(type, params) {
98
- if (!this.namespace)
99
- throw new Error('namespace not set');
100
- return key_1.KeyService.mintKey(this.namespace, type, params);
101
- }
102
- invalidateCache() {
103
- this.cache.invalidate();
104
- }
105
- async reserveScoutRole(scoutType, delay = enums_1.HMSH_SCOUT_INTERVAL_SECONDS) {
106
- const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
107
- appId: this.appId,
108
- scoutType,
109
- });
110
- const success = await this.exec('SET', key, `${scoutType}:${(0, utils_1.formatISODate)(new Date())}`, 'NX', 'EX', `${delay - 1}`);
111
- return this.isSuccessful(success);
112
- }
113
- async releaseScoutRole(scoutType) {
114
- const key = this.mintKey(key_1.KeyType.WORK_ITEMS, {
115
- appId: this.appId,
116
- scoutType,
117
- });
118
- const success = await this.exec('DEL', key);
119
- return this.isSuccessful(success);
120
- }
121
- async getSettings(bCreate = false) {
122
- let settings = this.cache?.getSettings();
123
- if (settings) {
124
- return settings;
125
- }
126
- else {
127
- if (bCreate) {
128
- const packageJson = await Promise.resolve().then(() => __importStar(require('../../package.json')));
129
- const version = packageJson['version'] || '0.0.0';
130
- settings = { namespace: key_1.HMNS, version };
131
- await this.setSettings(settings);
132
- return settings;
133
- }
134
- }
135
- throw new Error('settings not found');
136
- }
137
- async setSettings(manifest) {
138
- const params = {};
139
- const key = this.mintKey(key_1.KeyType.HOTMESH, params);
140
- return await this.redisClient[this.commands.hset](key, manifest);
141
- }
142
- async reserveSymbolRange(target, size, type, tryCount = 1) {
143
- const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
144
- const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
145
- activityId: target,
146
- appId: this.appId,
147
- });
148
- const response = await this.redisClient[this.commands.hsetnx](rangeKey, target, '?:?');
149
- if (response) {
150
- const upperLimit = await this.redisClient[this.commands.hincrby](rangeKey, ':cursor', size);
151
- const lowerLimit = upperLimit - size;
152
- const inclusiveRange = `${lowerLimit}:${upperLimit - 1}`;
153
- await this.redisClient[this.commands.hset](rangeKey, target, inclusiveRange);
154
- const metadataSeeds = this.seedSymbols(target, type, lowerLimit);
155
- await this.redisClient[this.commands.hset](symbolKey, metadataSeeds);
156
- return [lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS, upperLimit - 1, {}];
157
- }
158
- else {
159
- const range = await this.redisClient[this.commands.hget](rangeKey, target);
160
- const [lowerLimitString] = range.split(':');
161
- if (lowerLimitString === '?') {
162
- await (0, utils_1.sleepFor)(tryCount * 1000);
163
- if (tryCount < 5) {
164
- return this.reserveSymbolRange(target, size, type, tryCount + 1);
165
- }
166
- else {
167
- throw new Error('Symbol range reservation failed due to deployment contention');
168
- }
169
- }
170
- else {
171
- const lowerLimit = parseInt(lowerLimitString, 10);
172
- const symbols = await this.redisClient[this.commands.hgetall](symbolKey);
173
- const symbolCount = Object.keys(symbols).length;
174
- const actualLowerLimit = lowerLimit + serializer_1.MDATA_SYMBOLS.SLOTS + symbolCount;
175
- const upperLimit = Number(lowerLimit + size - 1);
176
- return [actualLowerLimit, upperLimit, symbols];
177
- }
178
- }
179
- }
180
- async getAllSymbols() {
181
- const rangeKey = this.mintKey(key_1.KeyType.SYMKEYS, { appId: this.appId });
182
- const ranges = await this.redisClient[this.commands.hgetall](rangeKey);
183
- const rangeKeys = Object.keys(ranges).sort();
184
- delete rangeKeys[':cursor'];
185
- const multi = this.getMulti();
186
- for (const rangeKey of rangeKeys) {
187
- const symbolKey = this.mintKey(key_1.KeyType.SYMKEYS, {
188
- activityId: rangeKey,
189
- appId: this.appId,
190
- });
191
- multi[this.commands.hgetall](symbolKey);
192
- }
193
- const results = (await multi.exec());
194
- const symbolSets = {};
195
- results.forEach((result, index) => {
196
- if (result) {
197
- let vals;
198
- if (Array.isArray(result) && result.length === 2) {
199
- vals = result[1];
200
- }
201
- else {
202
- vals = result;
203
- }
204
- for (const [key, value] of Object.entries(vals)) {
205
- symbolSets[value] = key.startsWith(rangeKeys[index])
206
- ? key
207
- : `${rangeKeys[index]}/${key}`;
208
- }
209
- }
210
- });
211
- return symbolSets;
212
- }
213
- async getSymbols(activityId) {
214
- let symbols = this.cache.getSymbols(this.appId, activityId);
215
- if (symbols) {
216
- return symbols;
217
- }
218
- else {
219
- const params = { activityId, appId: this.appId };
220
- const key = this.mintKey(key_1.KeyType.SYMKEYS, params);
221
- symbols = (await this.redisClient[this.commands.hgetall](key));
222
- this.cache.setSymbols(this.appId, activityId, symbols);
223
- return symbols;
224
- }
225
- }
226
- async addSymbols(activityId, symbols) {
227
- if (!symbols || !Object.keys(symbols).length)
228
- return false;
229
- const params = { activityId, appId: this.appId };
230
- const key = this.mintKey(key_1.KeyType.SYMKEYS, params);
231
- const success = await this.redisClient[this.commands.hset](key, symbols);
232
- this.cache.deleteSymbols(this.appId, activityId);
233
- return success > 0;
234
- }
235
- seedSymbols(target, type, startIndex) {
236
- if (type === 'JOB') {
237
- return this.seedJobSymbols(startIndex);
238
- }
239
- return this.seedActivitySymbols(startIndex, target);
240
- }
241
- seedJobSymbols(startIndex) {
242
- const hash = {};
243
- serializer_1.MDATA_SYMBOLS.JOB.KEYS.forEach((key) => {
244
- hash[`metadata/${key}`] = (0, utils_1.getSymKey)(startIndex);
245
- startIndex++;
246
- });
247
- return hash;
248
- }
249
- seedActivitySymbols(startIndex, activityId) {
250
- const hash = {};
251
- serializer_1.MDATA_SYMBOLS.ACTIVITY.KEYS.forEach((key) => {
252
- hash[`${activityId}/output/metadata/${key}`] = (0, utils_1.getSymKey)(startIndex);
253
- startIndex++;
254
- });
255
- return hash;
256
- }
257
- async getSymbolValues() {
258
- let symvals = this.cache.getSymbolValues(this.appId);
259
- if (symvals) {
260
- return symvals;
261
- }
262
- else {
263
- const key = this.mintKey(key_1.KeyType.SYMVALS, { appId: this.appId });
264
- symvals = await this.redisClient[this.commands.hgetall](key);
265
- this.cache.setSymbolValues(this.appId, symvals);
266
- return symvals;
267
- }
268
- }
269
- async addSymbolValues(symvals) {
270
- if (!symvals || !Object.keys(symvals).length)
271
- return false;
272
- const key = this.mintKey(key_1.KeyType.SYMVALS, { appId: this.appId });
273
- const success = await this.redisClient[this.commands.hset](key, symvals);
274
- this.cache.deleteSymbolValues(this.appId);
275
- return this.isSuccessful(success);
276
- }
277
- async getSymbolKeys(symbolNames) {
278
- const symbolLookups = [];
279
- for (const symbolName of symbolNames) {
280
- symbolLookups.push(this.getSymbols(symbolName));
281
- }
282
- const symbolSets = await Promise.all(symbolLookups);
283
- const symKeys = {};
284
- for (const symbolName of symbolNames) {
285
- symKeys[symbolName] = symbolSets.shift();
286
- }
287
- return symKeys;
288
- }
289
- async getApp(id, refresh = false) {
290
- let app = this.cache.getApp(id);
291
- if (refresh || !(app && Object.keys(app).length > 0)) {
292
- const params = { appId: id };
293
- const key = this.mintKey(key_1.KeyType.APP, params);
294
- const sApp = await this.redisClient[this.commands.hgetall](key);
295
- if (!sApp)
296
- return null;
297
- app = {};
298
- for (const field in sApp) {
299
- try {
300
- if (field === 'active') {
301
- app[field] = sApp[field] === 'true';
302
- }
303
- else {
304
- app[field] = sApp[field];
305
- }
306
- }
307
- catch (e) {
308
- app[field] = sApp[field];
309
- }
310
- }
311
- this.cache.setApp(id, app);
312
- }
313
- return app;
314
- }
315
- async setApp(id, version) {
316
- const params = { appId: id };
317
- const key = this.mintKey(key_1.KeyType.APP, params);
318
- const versionId = `versions/${version}`;
319
- const payload = {
320
- id,
321
- version,
322
- [versionId]: `deployed:${(0, utils_1.formatISODate)(new Date())}`,
323
- };
324
- await this.redisClient[this.commands.hset](key, payload);
325
- this.cache.setApp(id, payload);
326
- return payload;
327
- }
328
- async activateAppVersion(id, version) {
329
- const params = { appId: id };
330
- const key = this.mintKey(key_1.KeyType.APP, params);
331
- const versionId = `versions/${version}`;
332
- const app = await this.getApp(id, true);
333
- if (app && app[versionId]) {
334
- const payload = {
335
- id,
336
- version: version.toString(),
337
- [versionId]: `activated:${(0, utils_1.formatISODate)(new Date())}`,
338
- active: true,
339
- };
340
- Object.entries(payload).forEach(([key, value]) => {
341
- payload[key] = value.toString();
342
- });
343
- await this.redisClient[this.commands.hset](key, payload);
344
- return true;
345
- }
346
- throw new Error(`Version ${version} does not exist for app ${id}`);
347
- }
348
- async registerAppVersion(appId, version) {
349
- const params = { appId };
350
- const key = this.mintKey(key_1.KeyType.APP, params);
351
- const payload = {
352
- id: appId,
353
- version: version.toString(),
354
- [`versions/${version}`]: (0, utils_1.formatISODate)(new Date()),
355
- };
356
- return await this.redisClient[this.commands.hset](key, payload);
357
- }
358
- async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
359
- const params = {
360
- appId: appVersion.id,
361
- jobId,
362
- jobKey,
363
- dateTime,
364
- };
365
- const privateMulti = multi || this.getMulti();
366
- if (stats.general.length) {
367
- const generalStatsKey = this.mintKey(key_1.KeyType.JOB_STATS_GENERAL, params);
368
- for (const { target, value } of stats.general) {
369
- privateMulti[this.commands.hincrbyfloat](generalStatsKey, target, value);
370
- }
371
- }
372
- for (const { target, value } of stats.index) {
373
- const indexParams = { ...params, facet: target };
374
- const indexStatsKey = this.mintKey(key_1.KeyType.JOB_STATS_INDEX, indexParams);
375
- privateMulti[this.commands.rpush](indexStatsKey, value.toString());
376
- }
377
- for (const { target, value } of stats.median) {
378
- const medianParams = { ...params, facet: target };
379
- const medianStatsKey = this.mintKey(key_1.KeyType.JOB_STATS_MEDIAN, medianParams);
380
- this.zAdd(medianStatsKey, value, target, privateMulti);
381
- }
382
- if (!multi) {
383
- return await privateMulti.exec();
384
- }
385
- }
386
- hGetAllResult(result) {
387
- return result;
388
- }
389
- async getJobStats(jobKeys) {
390
- const multi = this.getMulti();
391
- for (const jobKey of jobKeys) {
392
- multi[this.commands.hgetall](jobKey);
393
- }
394
- const results = await multi.exec();
395
- const output = {};
396
- for (const [index, result] of results.entries()) {
397
- const key = jobKeys[index];
398
- const statsHash = this.hGetAllResult(result);
399
- if (statsHash && Object.keys(statsHash).length > 0) {
400
- const resolvedStatsHash = { ...statsHash };
401
- for (const [key, val] of Object.entries(resolvedStatsHash)) {
402
- resolvedStatsHash[key] = Number(val);
403
- }
404
- output[key] = resolvedStatsHash;
405
- }
406
- else {
407
- output[key] = {};
408
- }
409
- }
410
- return output;
411
- }
412
- async getJobIds(indexKeys, idRange) {
413
- const multi = this.getMulti();
414
- for (const idsKey of indexKeys) {
415
- multi[this.commands.lrange](idsKey, idRange[0], idRange[1]);
416
- }
417
- const results = await multi.exec();
418
- const output = {};
419
- for (const [index, result] of results.entries()) {
420
- const key = indexKeys[index];
421
- const idsList = result[1] || result;
422
- if (idsList && idsList.length > 0) {
423
- output[key] = idsList;
424
- }
425
- else {
426
- output[key] = [];
427
- }
428
- }
429
- return output;
430
- }
431
- async setStatus(collationKeyStatus, jobId, appId, multi) {
432
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
433
- return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, ':', collationKeyStatus);
434
- }
435
- async getStatus(jobId, appId) {
436
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
437
- const status = await this.redisClient[this.commands.hget](jobKey, ':');
438
- if (status === null) {
439
- throw new Error(`Job ${jobId} not found`);
440
- }
441
- return Number(status);
442
- }
443
- async setState({ ...state }, status, jobId, symbolNames, dIds, multi) {
444
- delete state['metadata/js'];
445
- const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, {
446
- appId: this.appId,
447
- jobId,
448
- });
449
- const symKeys = await this.getSymbolKeys(symbolNames);
450
- const symVals = await this.getSymbolValues();
451
- this.serializer.resetSymbols(symKeys, symVals, dIds);
452
- const hashData = this.serializer.package(state, symbolNames);
453
- if (status !== null) {
454
- hashData[':'] = status.toString();
455
- }
456
- else {
457
- delete hashData[':'];
458
- }
459
- await (multi || this.redisClient)[this.commands.hset](hashKey, hashData);
460
- return jobId;
461
- }
462
- async getQueryState(jobId, fields) {
463
- const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
464
- const _fields = fields.map((field) => {
465
- if (field.startsWith('"')) {
466
- return field.slice(1, -1);
467
- }
468
- return `_${field}`;
469
- });
470
- const jobDataArray = await this.redisClient[this.commands.hmget](key, _fields);
471
- const jobData = {};
472
- fields.forEach((field, index) => {
473
- if (field.startsWith('"')) {
474
- field = field.slice(1, -1);
475
- }
476
- jobData[field] = jobDataArray[index];
477
- });
478
- return jobData;
479
- }
480
- async getState(jobId, consumes, dIds) {
481
- const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
482
- const symbolNames = Object.keys(consumes);
483
- const symKeys = await this.getSymbolKeys(symbolNames);
484
- this.serializer.resetSymbols(symKeys, {}, dIds);
485
- const fields = this.serializer.abbreviate(consumes, symbolNames, [':']);
486
- const jobDataArray = await this.redisClient[this.commands.hmget](key, fields);
487
- const jobData = {};
488
- let atLeast1 = false;
489
- fields.forEach((field, index) => {
490
- if (jobDataArray[index]) {
491
- atLeast1 = true;
492
- }
493
- jobData[field] = jobDataArray[index];
494
- });
495
- if (atLeast1) {
496
- const symVals = await this.getSymbolValues();
497
- this.serializer.resetSymbols(symKeys, symVals, dIds);
498
- const state = this.serializer.unpackage(jobData, symbolNames);
499
- let status = 0;
500
- if (state[':']) {
501
- status = Number(state[':']);
502
- state[`metadata/js`] = status;
503
- delete state[':'];
504
- }
505
- return [state, status];
506
- }
507
- else {
508
- throw new errors_1.GetStateError(jobId);
509
- }
510
- }
511
- async getRaw(jobId) {
512
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
513
- appId: this.appId,
514
- jobId,
515
- });
516
- const job = await this.redisClient[this.commands.hgetall](jobKey);
517
- if (!job) {
518
- throw new errors_1.GetStateError(jobId);
519
- }
520
- return job;
521
- }
522
- async collate(jobId, activityId, amount, dIds, multi) {
523
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
524
- appId: this.appId,
525
- jobId,
526
- });
527
- const collationKey = `${activityId}/output/metadata/as`;
528
- const symbolNames = [activityId];
529
- const symKeys = await this.getSymbolKeys(symbolNames);
530
- const symVals = await this.getSymbolValues();
531
- this.serializer.resetSymbols(symKeys, symVals, dIds);
532
- const payload = { [collationKey]: amount.toString() };
533
- const hashData = this.serializer.package(payload, symbolNames);
534
- const targetId = Object.keys(hashData)[0];
535
- return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, targetId, amount);
536
- }
537
- async collateSynthetic(jobId, guid, amount, multi) {
538
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
539
- appId: this.appId,
540
- jobId,
541
- });
542
- return await (multi || this.redisClient)[this.commands.hincrbyfloat](jobKey, guid, amount.toString());
543
- }
544
- async setStateNX(jobId, appId, status) {
545
- const hashKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId, jobId });
546
- const result = await this.redisClient[this.commands.hsetnx](hashKey, ':', status?.toString() ?? '1');
547
- return this.isSuccessful(result);
548
- }
549
- async getSchema(activityId, appVersion) {
550
- const schema = this.cache.getSchema(appVersion.id, appVersion.version, activityId);
551
- if (schema) {
552
- return schema;
553
- }
554
- else {
555
- const schemas = await this.getSchemas(appVersion);
556
- return schemas[activityId];
557
- }
558
- }
559
- async getSchemas(appVersion) {
560
- let schemas = this.cache.getSchemas(appVersion.id, appVersion.version);
561
- if (schemas && Object.keys(schemas).length > 0) {
562
- return schemas;
563
- }
564
- else {
565
- const params = {
566
- appId: appVersion.id,
567
- appVersion: appVersion.version,
568
- };
569
- const key = this.mintKey(key_1.KeyType.SCHEMAS, params);
570
- schemas = {};
571
- const hash = await this.redisClient[this.commands.hgetall](key);
572
- Object.entries(hash).forEach(([key, value]) => {
573
- schemas[key] = JSON.parse(value);
574
- });
575
- this.cache.setSchemas(appVersion.id, appVersion.version, schemas);
576
- return schemas;
577
- }
578
- }
579
- async setSchemas(schemas, appVersion) {
580
- const params = {
581
- appId: appVersion.id,
582
- appVersion: appVersion.version,
583
- };
584
- const key = this.mintKey(key_1.KeyType.SCHEMAS, params);
585
- const _schemas = { ...schemas };
586
- Object.entries(_schemas).forEach(([key, value]) => {
587
- _schemas[key] = JSON.stringify(value);
588
- });
589
- const response = await this.redisClient[this.commands.hset](key, _schemas);
590
- this.cache.setSchemas(appVersion.id, appVersion.version, schemas);
591
- return response;
592
- }
593
- async setSubscriptions(subscriptions, appVersion) {
594
- const params = {
595
- appId: appVersion.id,
596
- appVersion: appVersion.version,
597
- };
598
- const key = this.mintKey(key_1.KeyType.SUBSCRIPTIONS, params);
599
- const _subscriptions = { ...subscriptions };
600
- Object.entries(_subscriptions).forEach(([key, value]) => {
601
- _subscriptions[key] = JSON.stringify(value);
602
- });
603
- const status = await this.redisClient[this.commands.hset](key, _subscriptions);
604
- this.cache.setSubscriptions(appVersion.id, appVersion.version, subscriptions);
605
- return this.isSuccessful(status);
606
- }
607
- async getSubscriptions(appVersion) {
608
- let subscriptions = this.cache.getSubscriptions(appVersion.id, appVersion.version);
609
- if (subscriptions && Object.keys(subscriptions).length > 0) {
610
- return subscriptions;
611
- }
612
- else {
613
- const params = {
614
- appId: appVersion.id,
615
- appVersion: appVersion.version,
616
- };
617
- const key = this.mintKey(key_1.KeyType.SUBSCRIPTIONS, params);
618
- subscriptions =
619
- await this.redisClient[this.commands.hgetall](key) || {};
620
- Object.entries(subscriptions).forEach(([key, value]) => {
621
- subscriptions[key] = JSON.parse(value);
622
- });
623
- this.cache.setSubscriptions(appVersion.id, appVersion.version, subscriptions);
624
- return subscriptions;
625
- }
626
- }
627
- async getSubscription(topic, appVersion) {
628
- const subscriptions = await this.getSubscriptions(appVersion);
629
- return subscriptions[topic];
630
- }
631
- async setTransitions(transitions, appVersion) {
632
- const params = {
633
- appId: appVersion.id,
634
- appVersion: appVersion.version,
635
- };
636
- const key = this.mintKey(key_1.KeyType.SUBSCRIPTION_PATTERNS, params);
637
- const _subscriptions = { ...transitions };
638
- Object.entries(_subscriptions).forEach(([key, value]) => {
639
- _subscriptions[key] = JSON.stringify(value);
640
- });
641
- if (Object.keys(_subscriptions).length !== 0) {
642
- const response = await this.redisClient[this.commands.hset](key, _subscriptions);
643
- this.cache.setTransitions(appVersion.id, appVersion.version, transitions);
644
- return response;
645
- }
646
- }
647
- async getTransitions(appVersion) {
648
- let transitions = this.cache.getTransitions(appVersion.id, appVersion.version);
649
- if (transitions && Object.keys(transitions).length > 0) {
650
- return transitions;
651
- }
652
- else {
653
- const params = {
654
- appId: appVersion.id,
655
- appVersion: appVersion.version,
656
- };
657
- const key = this.mintKey(key_1.KeyType.SUBSCRIPTION_PATTERNS, params);
658
- transitions = {};
659
- const hash = await this.redisClient[this.commands.hgetall](key);
660
- Object.entries(hash).forEach(([key, value]) => {
661
- transitions[key] = JSON.parse(value);
662
- });
663
- this.cache.setTransitions(appVersion.id, appVersion.version, transitions);
664
- return transitions;
665
- }
666
- }
667
- async setHookRules(hookRules) {
668
- const key = this.mintKey(key_1.KeyType.HOOKS, { appId: this.appId });
669
- const _hooks = {};
670
- Object.entries(hookRules).forEach(([key, value]) => {
671
- _hooks[key.toString()] = JSON.stringify(value);
672
- });
673
- if (Object.keys(_hooks).length !== 0) {
674
- const response = await this.redisClient[this.commands.hset](key, _hooks);
675
- this.cache.setHookRules(this.appId, hookRules);
676
- return response;
677
- }
678
- }
679
- async getHookRules() {
680
- let patterns = this.cache.getHookRules(this.appId);
681
- if (patterns && Object.keys(patterns).length > 0) {
682
- return patterns;
683
- }
684
- else {
685
- const key = this.mintKey(key_1.KeyType.HOOKS, { appId: this.appId });
686
- const _hooks = await this.redisClient[this.commands.hgetall](key);
687
- patterns = {};
688
- Object.entries(_hooks).forEach(([key, value]) => {
689
- patterns[key] = JSON.parse(value);
690
- });
691
- this.cache.setHookRules(this.appId, patterns);
692
- return patterns;
693
- }
694
- }
695
- async setHookSignal(hook, multi) {
696
- const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
697
- const { topic, resolved, jobId } = hook;
698
- const signalKey = `${topic}:${resolved}`;
699
- await this.setnxex(`${key}:${signalKey}`, jobId, Math.max(hook.expire, enums_1.HMSH_SIGNAL_EXPIRE));
700
- }
701
- async getHookSignal(topic, resolved) {
702
- const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
703
- const response = await this.redisClient[this.commands.get](`${key}:${topic}:${resolved}`);
704
- return response ? response.toString() : undefined;
705
- }
706
- async deleteHookSignal(topic, resolved) {
707
- const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
708
- const response = await this.redisClient[this.commands.del](`${key}:${topic}:${resolved}`);
709
- return response ? Number(response) : undefined;
710
- }
711
- async addTaskQueues(keys) {
712
- const multi = this.getMulti();
713
- const zsetKey = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId });
714
- for (const key of keys) {
715
- multi[this.commands.zadd](zsetKey, { score: Date.now().toString(), value: key }, { NX: true });
716
- }
717
- await multi.exec();
718
- }
719
- async getActiveTaskQueue() {
720
- let workItemKey = this.cache.getActiveTaskQueue(this.appId) || null;
721
- if (!workItemKey) {
722
- const zsetKey = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId });
723
- const result = await this.redisClient[this.commands.zrange](zsetKey, 0, 0);
724
- workItemKey = result.length > 0 ? result[0] : null;
725
- if (workItemKey) {
726
- this.cache.setWorkItem(this.appId, workItemKey);
727
- }
728
- }
729
- return workItemKey;
730
- }
731
- async deleteProcessedTaskQueue(workItemKey, key, processedKey, scrub = false) {
732
- const zsetKey = this.mintKey(key_1.KeyType.WORK_ITEMS, { appId: this.appId });
733
- const didRemove = await this.redisClient[this.commands.zrem](zsetKey, workItemKey);
734
- if (didRemove) {
735
- if (scrub) {
736
- this.redisClient[this.commands.expire](processedKey, 0);
737
- this.redisClient[this.commands.expire](key.split(':').slice(0, 5).join(':'), 0);
738
- }
739
- else {
740
- await this.redisClient[this.commands.rename](processedKey, key);
741
- }
742
- }
743
- this.cache.removeWorkItem(this.appId);
744
- }
745
- async processTaskQueue(sourceKey, destinationKey) {
746
- return await this.redisClient[this.commands.lmove](sourceKey, destinationKey, 'LEFT', 'RIGHT');
747
- }
748
- async expireJob(jobId, inSeconds, redisMulti) {
749
- if (!isNaN(inSeconds) && inSeconds > 0) {
750
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
751
- appId: this.appId,
752
- jobId,
753
- });
754
- await (redisMulti || this.redisClient)[this.commands.expire](jobKey, inSeconds);
755
- }
756
- }
757
- async getDependencies(jobId) {
758
- const depParams = { appId: this.appId, jobId };
759
- const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
760
- return this.redisClient[this.commands.lrange](depKey, 0, -1);
761
- }
762
- async registerTimeHook(jobId, gId, activityId, type, deletionTime, dad, multi) {
763
- const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, {
764
- appId: this.appId,
765
- timeValue: deletionTime,
766
- });
767
- const timeEvent = [type, activityId, gId, dad, jobId].join(key_1.VALSEP);
768
- const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
769
- if (multi || len === 1) {
770
- const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
771
- await this.zAdd(zsetKey, deletionTime.toString(), listKey, multi);
772
- }
773
- }
774
- async getNextTask(listKey) {
775
- const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
776
- listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
777
- if (listKey) {
778
- let [pType, pKey] = this.resolveTaskKeyContext(listKey);
779
- const timeEvent = await this.redisClient[this.commands.lpop](pKey);
780
- if (timeEvent) {
781
- let [type, activityId, gId, _pd, ...jobId] = timeEvent.split(key_1.VALSEP);
782
- const jid = jobId.join(key_1.VALSEP);
783
- if (type === 'delist') {
784
- pType = 'delist';
785
- }
786
- else if (type === 'child') {
787
- pType = 'child';
788
- }
789
- else if (type === 'expire-child') {
790
- type = 'expire';
791
- }
792
- return [listKey, jid, gId, activityId, pType];
793
- }
794
- await this.redisClient[this.commands.zrem](zsetKey, listKey);
795
- return true;
796
- }
797
- return false;
798
- }
799
- resolveTaskKeyContext(listKey) {
800
- if (listKey.startsWith(`${key_1.TYPSEP}INTERRUPT`)) {
801
- return ['interrupt', listKey.split(key_1.TYPSEP)[2]];
802
- }
803
- else if (listKey.startsWith(`${key_1.TYPSEP}EXPIRE`)) {
804
- return ['expire', listKey.split(key_1.TYPSEP)[2]];
805
- }
806
- else {
807
- return ['sleep', listKey];
808
- }
809
- }
810
- async interrupt(topic, jobId, options = {}) {
811
- try {
812
- const status = await this.getStatus(jobId, this.appId);
813
- if (status <= 0) {
814
- throw new Error(`Job ${jobId} already completed`);
815
- }
816
- const amount = -1000000000;
817
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
818
- appId: this.appId,
819
- jobId,
820
- });
821
- const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
822
- if (result <= amount) {
823
- throw new Error(`Job ${jobId} already completed`);
824
- }
825
- if (options.throw !== false) {
826
- const errKey = `metadata/err`;
827
- const symbolNames = [`$${topic}`];
828
- const symKeys = await this.getSymbolKeys(symbolNames);
829
- const symVals = await this.getSymbolValues();
830
- this.serializer.resetSymbols(symKeys, symVals, {});
831
- const err = JSON.stringify({
832
- code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
833
- message: options.reason ?? `job [${jobId}] interrupted`,
834
- stack: options.stack ?? '',
835
- job_id: jobId,
836
- });
837
- const payload = { [errKey]: amount.toString() };
838
- const hashData = this.serializer.package(payload, symbolNames);
839
- const errSymbol = Object.keys(hashData)[0];
840
- await this.redisClient[this.commands.hset](jobKey, errSymbol, err);
841
- }
842
- }
843
- catch (e) {
844
- if (!options.suppress) {
845
- throw e;
846
- }
847
- else {
848
- this.logger.debug('suppressed-interrupt', { message: e.message });
849
- }
850
- }
851
- }
852
- async scrub(jobId) {
853
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
854
- appId: this.appId,
855
- jobId,
856
- });
857
- await this.redisClient[this.commands.del](jobKey);
858
- }
859
- async findJobs(queryString = '*', limit = 1000, batchSize = 1000, cursor = '0') {
860
- const matchKey = this.mintKey(key_1.KeyType.JOB_STATE, {
861
- appId: this.appId,
862
- jobId: queryString,
863
- });
864
- let keys;
865
- const matchingKeys = [];
866
- do {
867
- const output = (await this.exec('SCAN', cursor, 'MATCH', matchKey, 'COUNT', batchSize.toString()));
868
- if (Array.isArray(output)) {
869
- [cursor, keys] = output;
870
- for (const key of [...keys]) {
871
- matchingKeys.push(key);
872
- }
873
- if (matchingKeys.length >= limit) {
874
- break;
875
- }
876
- }
877
- else {
878
- break;
879
- }
880
- } while (cursor !== '0');
881
- return [cursor, matchingKeys];
882
- }
883
- async findJobFields(jobId, fieldMatchPattern = '*', limit = 1000, batchSize = 1000, cursor = '0') {
884
- let fields = [];
885
- const matchingFields = {};
886
- const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, {
887
- appId: this.appId,
888
- jobId,
889
- });
890
- let len = 0;
891
- do {
892
- const output = (await this.exec('HSCAN', jobKey, cursor, 'MATCH', fieldMatchPattern, 'COUNT', batchSize.toString()));
893
- if (Array.isArray(output)) {
894
- [cursor, fields] = output;
895
- for (let i = 0; i < fields.length; i += 2) {
896
- len++;
897
- matchingFields[fields[i]] = fields[i + 1];
898
- }
899
- }
900
- else {
901
- break;
902
- }
903
- } while (cursor !== '0' && len < limit);
904
- return [cursor, matchingFields];
905
- }
906
- async setThrottleRate(options) {
907
- const key = this.mintKey(key_1.KeyType.THROTTLE_RATE, { appId: this.appId });
908
- if (options.guid) {
909
- return;
910
- }
911
- const rate = options.throttle.toString();
912
- if (options.topic) {
913
- await this.redisClient[this.commands.hset](key, {
914
- [options.topic]: rate,
915
- });
916
- }
917
- else {
918
- const multi = this.getMulti();
919
- multi[this.commands.del](key);
920
- multi[this.commands.hset](key, { ':': rate });
921
- await multi.exec();
922
- }
923
- }
924
- async getThrottleRates() {
925
- const key = this.mintKey(key_1.KeyType.THROTTLE_RATE, { appId: this.appId });
926
- const response = await this.redisClient[this.commands.hgetall](key);
927
- return response ?? {};
928
- }
929
- async getThrottleRate(topic) {
930
- const resolveRate = (response, topic) => {
931
- const rate = topic in response ? Number(response[topic]) : 0;
932
- if (isNaN(rate))
933
- return 0;
934
- if (rate == -1)
935
- return enums_1.MAX_DELAY;
936
- return Math.max(Math.min(rate, enums_1.MAX_DELAY), 0);
937
- };
938
- const response = await this.getThrottleRates();
939
- const globalRate = resolveRate(response, ':');
940
- if (topic === ':' || !(topic in response)) {
941
- return globalRate;
942
- }
943
- return resolveRate(response, topic);
5
+ constructor(client) {
6
+ this.storeClient = client;
944
7
  }
945
8
  }
946
9
  exports.StoreService = StoreService;