@cubejs-backend/query-orchestrator 1.3.42 → 1.3.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/src/orchestrator/LocalQueueDriver.d.ts +7 -123
  2. package/dist/src/orchestrator/LocalQueueDriver.d.ts.map +1 -1
  3. package/dist/src/orchestrator/LocalQueueDriver.js +12 -352
  4. package/dist/src/orchestrator/LocalQueueDriver.js.map +1 -1
  5. package/dist/src/orchestrator/LocalQueueDriverConnection.d.ts +70 -0
  6. package/dist/src/orchestrator/LocalQueueDriverConnection.d.ts.map +1 -0
  7. package/dist/src/orchestrator/LocalQueueDriverConnection.js +235 -0
  8. package/dist/src/orchestrator/LocalQueueDriverConnection.js.map +1 -0
  9. package/dist/src/orchestrator/PreAggregationLoadCache.d.ts +3 -0
  10. package/dist/src/orchestrator/PreAggregationLoadCache.d.ts.map +1 -1
  11. package/dist/src/orchestrator/PreAggregationLoader.d.ts.map +1 -1
  12. package/dist/src/orchestrator/PreAggregationLoader.js +3 -1
  13. package/dist/src/orchestrator/PreAggregationLoader.js.map +1 -1
  14. package/dist/src/orchestrator/PreAggregations.d.ts +2 -3
  15. package/dist/src/orchestrator/PreAggregations.d.ts.map +1 -1
  16. package/dist/src/orchestrator/PreAggregations.js +2 -7
  17. package/dist/src/orchestrator/PreAggregations.js.map +1 -1
  18. package/dist/src/orchestrator/QueryCache.d.ts +4 -4
  19. package/dist/src/orchestrator/QueryCache.d.ts.map +1 -1
  20. package/dist/src/orchestrator/QueryCache.js +49 -50
  21. package/dist/src/orchestrator/QueryCache.js.map +1 -1
  22. package/dist/src/orchestrator/QueryOrchestrator.d.ts +5 -6
  23. package/dist/src/orchestrator/QueryOrchestrator.d.ts.map +1 -1
  24. package/dist/src/orchestrator/QueryOrchestrator.js +0 -16
  25. package/dist/src/orchestrator/QueryOrchestrator.js.map +1 -1
  26. package/dist/src/orchestrator/QueryQueue.d.ts +72 -160
  27. package/dist/src/orchestrator/QueryQueue.d.ts.map +1 -1
  28. package/dist/src/orchestrator/QueryQueue.js +51 -160
  29. package/dist/src/orchestrator/QueryQueue.js.map +1 -1
  30. package/dist/src/orchestrator/index.d.ts +1 -0
  31. package/dist/src/orchestrator/index.d.ts.map +1 -1
  32. package/dist/src/orchestrator/index.js +1 -0
  33. package/dist/src/orchestrator/index.js.map +1 -1
  34. package/package.json +6 -6
  35. package/dist/src/orchestrator/BaseQueueEventsBus.d.ts +0 -6
  36. package/dist/src/orchestrator/BaseQueueEventsBus.d.ts.map +0 -1
  37. package/dist/src/orchestrator/BaseQueueEventsBus.js +0 -14
  38. package/dist/src/orchestrator/BaseQueueEventsBus.js.map +0 -1
  39. package/dist/src/orchestrator/LocalQueueEventsBus.d.ts +0 -5
  40. package/dist/src/orchestrator/LocalQueueEventsBus.d.ts.map +0 -1
  41. package/dist/src/orchestrator/LocalQueueEventsBus.js +0 -12
  42. package/dist/src/orchestrator/LocalQueueEventsBus.js.map +0 -1
@@ -1,125 +1,9 @@
1
- /**
2
- * @implements {QueueDriverConnectionInterface}
3
- */
4
- export class LocalQueueDriverConnection implements QueueDriverConnectionInterface {
5
- constructor(driver: any, options: any);
6
- redisQueuePrefix: any;
7
- continueWaitTimeout: any;
8
- heartBeatTimeout: any;
9
- concurrency: any;
10
- driver: any;
11
- results: any;
12
- resultPromises: any;
13
- queryDef: any;
14
- toProcess: any;
15
- recent: any;
16
- active: any;
17
- heartBeat: any;
18
- processingCounter: any;
19
- processingLocks: any;
20
- getQueueEventsBus: any;
21
- getQueriesToCancel(): Promise<QueryKeysTuple[]>;
22
- /**
23
- * @returns {Promise<GetActiveAndToProcessResponse>}
24
- */
25
- getActiveAndToProcess(): Promise<GetActiveAndToProcessResponse>;
26
- getResultPromise(resultListKey: any): any;
27
- getResultBlocking(queryKeyHash: any): Promise<any>;
28
- /**
29
- * Returns promise wich will be resolved with the specified by the
30
- * queryKey query result or null if query was not added to the
31
- * processing.
32
- * @param {*} queryKey
33
- * @returns {Promise<null | *>}
34
- */
35
- getResult(queryKey: any): Promise<null | any>;
36
- /**
37
- * @protected
38
- */
39
- protected queueArray(queueObj: any, orderFilterLessThan: any): any[];
40
- /**
41
- * @protected
42
- * @param queueObj
43
- * @param orderFilterLessThan
44
- * @returns {QueryKeysTuple[]}
45
- */
46
- protected queueArrayAsTuple(queueObj: any, orderFilterLessThan: any): QueryKeysTuple[];
47
- /**
48
- * Adds specified by the queryKey query to the queue, returns tuple
49
- * with the operation result.
50
- *
51
- * @typedef {[added: number, _b: null, _c: null, toProcessLength: number, addedTime: number]} AddedTuple
52
- *
53
- * @param {number} keyScore
54
- * @param {*} queryKey
55
- * @param {number} orphanedTime
56
- * @param {string} queryHandler (for the regular query is eq to 'query')
57
- * @param {*} query
58
- * @param {number} priority
59
- * @param {*} options
60
- * @returns {AddedTuple}
61
- */
62
- addToQueue(keyScore: number, queryKey: any, orphanedTime: number, queryHandler: string, query: any, priority: number, options: any): [added: number, _b: null, _c: null, toProcessLength: number, addedTime: number];
63
- getToProcessQueries(): QueryKeysTuple[];
64
- getActiveQueries(): QueryKeysTuple[];
65
- getQueryAndRemove(queryKey: any): Promise<any[]>;
66
- cancelQuery(queryKey: any): Promise<any>;
67
- setResultAndRemoveQuery(queryKey: any, executionResult: any, processingId: any): Promise<boolean>;
68
- getNextProcessingId(): any;
69
- getOrphanedQueries(): QueryKeysTuple[];
70
- getStalledQueries(): QueryKeysTuple[];
71
- getQueryStageState(onlyKeys: any): Promise<any[]>;
72
- getQueryDef(queryKey: any): Promise<any>;
73
- /**
74
- * Updates heart beat for the processing query by its `queryKey`.
75
- *
76
- * @param {string} queryKey
77
- */
78
- updateHeartBeat(queryKey: string): void;
79
- retrieveForProcessing(queryKey: any, processingId: any): any[];
80
- freeProcessingLock(queryKey: any, processingId: any, activated: any): void;
81
- optimisticQueryUpdate(queryKey: any, toUpdate: any, processingId: any): Promise<boolean>;
82
- release(): void;
83
- /**
84
- * Returns cache key to the specified by the queryKey query and the
85
- * specified by the suffix query state.
86
- * @param {*} queryKey
87
- * @param {string} suffix
88
- * @returns {string}
89
- */
90
- queryRedisKey(queryKey: any, suffix: string): string;
91
- /**
92
- * Returns cache key to the cached query result.
93
- * @param {*} queryKey
94
- * @returns {string}
95
- */
96
- resultListKey(queryKey: any): string;
97
- /**
98
- * Returns hash sum of the query specified by the queryKey.
99
- * @param {*} queryKey
100
- * @returns {string}
101
- */
102
- redisHash(queryKey: any): string;
103
- }
104
- /**
105
- * @implements {QueueDriverInterface}
106
- */
107
- export class LocalQueueDriver extends BaseQueueDriver implements QueueDriverInterface {
108
- constructor(options: any);
109
- options: any;
110
- results: any;
111
- resultPromises: any;
112
- queryDef: any;
113
- toProcess: any;
114
- recent: any;
115
- active: any;
116
- heartBeat: any;
117
- processingCounter: any;
118
- processingLocks: any;
119
- createConnection(): LocalQueueDriverConnection;
120
- release(client: any): void;
121
- }
122
- import { QueueDriverConnectionInterface } from '@cubejs-backend/base-driver';
123
- import { QueueDriverInterface } from '@cubejs-backend/base-driver';
1
+ import { QueueDriverConnectionInterface, QueueDriverOptions } from '@cubejs-backend/base-driver';
124
2
  import { BaseQueueDriver } from './BaseQueueDriver';
3
+ export declare class LocalQueueDriver extends BaseQueueDriver {
4
+ private readonly options;
5
+ constructor(options: QueueDriverOptions);
6
+ createConnection(): Promise<QueueDriverConnectionInterface>;
7
+ release(client: QueueDriverConnectionInterface): void;
8
+ }
125
9
  //# sourceMappingURL=LocalQueueDriver.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"LocalQueueDriver.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/LocalQueueDriver.js"],"names":[],"mappings":"AAIA;;GAEG;AACH;IACE,uCAgBC;IAfC,sBAAgD;IAChD,yBAAsD;IACtD,sBAAgD;IAChD,iBAAsC;IACtC,YAAoB;IACpB,aAA6B;IAC7B,oBAA2C;IAC3C,cAA+B;IAC/B,eAAiC;IACjC,YAA2B;IAC3B,YAA2B;IAC3B,eAAiC;IACjC,uBAAiD;IACjD,qBAA6C;IAC7C,uBAAkD;IAGpD,gDAOC;IAED;;OAEG;IACH,yBAFa,sCAAsC,CAUlD;IAED,0CASC;IAED,mDAiBC;IAED;;;;;;OAMG;IACH,0BAFa,QAAQ,IAAI,MAAI,CAAC,CAS7B;IAED;;OAEG;IACH,qEAOC;IAED;;;;;OAKG;IACH,sEAFa,gBAAgB,CAS5B;IAED;;;;;;;;;;;;;;OAcG;IACH,qBATW,MAAM,+BAEN,MAAM,gBACN,MAAM,wBAEN,MAAM,iGAsDhB;IAED,wCAEC;IAED,qCAEC;IAED,iDAUC;IAED,yCAaC;IAED,kGAyBC;IAED,2BAGC;IAED,uCAEC;IAED,sCAEC;IAED,kDAEC;IAED,yCAEC;IAED;;;;OAIG;IACH,0BAFW,MAAM,QAOhB;IAED,+DAuCC;IAED,2EAQC;IAED,yFAOC;IAGD,gBACC;IAED;;;;;;OAMG;IACH,qCAHW,MAAM,GACJ,MAAM,CAIlB;IAED;;;;OAIG;IACH,8BAFa,MAAM,CAIlB;IAED;;;;OAIG;IACH,0BAFa,MAAM,CAIlB;CACF;AAYD;;GAEG;AACH;IACE,0BAqBC;IAnBC,aAAsB;IAUtB,aAAgD;IAChD,oBAA8D;IAC9D,cAAkD;IAClD,eAAoD;IACpD,YAA8C;IAC9C,YAA8C;IAC9C,eAAoD;IACpD,uBAAqE;IACrE,qBAAgE;IAGlE,+CAEC;IAED,2BAEC;CACF;+CApaoE,6BAA6B;qCAA7B,6BAA6B;gCAClE,mBAAmB"}
1
+ {"version":3,"file":"LocalQueueDriver.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/LocalQueueDriver.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,8BAA8B,EAC9B,kBAAkB,EACnB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAapD,qBAAa,gBAAiB,SAAQ,eAAe;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,OAAO,EAAE,kBAAkB;IAKjC,gBAAgB,IAAI,OAAO,CAAC,8BAA8B,CAAC;IAKjE,OAAO,CAAC,MAAM,EAAE,8BAA8B,GAAG,IAAI;CAG7D"}
@@ -1,364 +1,24 @@
1
1
  "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
2
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.LocalQueueDriver = exports.LocalQueueDriverConnection = void 0;
7
- const ramda_1 = __importDefault(require("ramda"));
8
- const base_driver_1 = require("@cubejs-backend/base-driver");
3
+ exports.LocalQueueDriver = void 0;
9
4
  const BaseQueueDriver_1 = require("./BaseQueueDriver");
10
- /**
11
- * @implements {QueueDriverConnectionInterface}
12
- */
13
- class LocalQueueDriverConnection {
14
- constructor(driver, options) {
15
- this.redisQueuePrefix = options.redisQueuePrefix;
16
- this.continueWaitTimeout = options.continueWaitTimeout;
17
- this.heartBeatTimeout = options.heartBeatTimeout;
18
- this.concurrency = options.concurrency;
19
- this.driver = driver;
20
- this.results = driver.results;
21
- this.resultPromises = driver.resultPromises;
22
- this.queryDef = driver.queryDef;
23
- this.toProcess = driver.toProcess;
24
- this.recent = driver.recent;
25
- this.active = driver.active;
26
- this.heartBeat = driver.heartBeat;
27
- this.processingCounter = driver.processingCounter;
28
- this.processingLocks = driver.processingLocks;
29
- this.getQueueEventsBus = options.getQueueEventsBus;
30
- }
31
- async getQueriesToCancel() {
32
- const [stalled, orphaned] = await Promise.all([
33
- this.getStalledQueries(),
34
- this.getOrphanedQueries(),
35
- ]);
36
- return stalled.concat(orphaned);
37
- }
38
- /**
39
- * @returns {Promise<GetActiveAndToProcessResponse>}
40
- */
41
- async getActiveAndToProcess() {
42
- const active = this.queueArrayAsTuple(this.active);
43
- const toProcess = this.queueArrayAsTuple(this.toProcess);
44
- return [
45
- active,
46
- toProcess
47
- ];
48
- }
49
- getResultPromise(resultListKey) {
50
- if (!this.resultPromises[resultListKey]) {
51
- let resolveMethod = null;
52
- this.resultPromises[resultListKey] = new Promise(resolve => {
53
- resolveMethod = resolve;
54
- });
55
- this.resultPromises[resultListKey].resolve = resolveMethod;
56
- }
57
- return this.resultPromises[resultListKey];
58
- }
59
- async getResultBlocking(queryKeyHash) {
60
- // Double redisHash apply is being used here
61
- const resultListKey = this.resultListKey(queryKeyHash);
62
- if (!this.queryDef[queryKeyHash] && !this.resultPromises[resultListKey]) {
63
- return null;
64
- }
65
- const timeoutPromise = (timeout) => new Promise((resolve) => setTimeout(() => resolve(null), timeout));
66
- const res = await Promise.race([
67
- this.getResultPromise(resultListKey),
68
- timeoutPromise(this.continueWaitTimeout * 1000),
69
- ]);
70
- if (res) {
71
- delete this.resultPromises[resultListKey];
72
- }
73
- return res;
74
- }
75
- /**
76
- * Returns promise wich will be resolved with the specified by the
77
- * queryKey query result or null if query was not added to the
78
- * processing.
79
- * @param {*} queryKey
80
- * @returns {Promise<null | *>}
81
- */
82
- async getResult(queryKey) {
83
- const resultListKey = this.resultListKey(queryKey);
84
- if (this.resultPromises[resultListKey] && this.resultPromises[resultListKey].resolved) {
85
- return this.getResultBlocking(queryKey);
86
- }
87
- return null;
88
- }
89
- /**
90
- * @protected
91
- */
92
- queueArray(queueObj, orderFilterLessThan) {
93
- return ramda_1.default.pipe(ramda_1.default.values, ramda_1.default.filter(orderFilterLessThan ? q => q.order < orderFilterLessThan : ramda_1.default.identity), ramda_1.default.sortBy(q => q.order), ramda_1.default.map(q => q.key))(queueObj);
94
- }
95
- /**
96
- * @protected
97
- * @param queueObj
98
- * @param orderFilterLessThan
99
- * @returns {QueryKeysTuple[]}
100
- */
101
- queueArrayAsTuple(queueObj, orderFilterLessThan) {
102
- return ramda_1.default.pipe(ramda_1.default.values, ramda_1.default.filter(orderFilterLessThan ? q => q.order < orderFilterLessThan : ramda_1.default.identity), ramda_1.default.sortBy(q => q.order), ramda_1.default.map(q => [q.key, q.queueId]))(queueObj);
103
- }
104
- /**
105
- * Adds specified by the queryKey query to the queue, returns tuple
106
- * with the operation result.
107
- *
108
- * @typedef {[added: number, _b: null, _c: null, toProcessLength: number, addedTime: number]} AddedTuple
109
- *
110
- * @param {number} keyScore
111
- * @param {*} queryKey
112
- * @param {number} orphanedTime
113
- * @param {string} queryHandler (for the regular query is eq to 'query')
114
- * @param {*} query
115
- * @param {number} priority
116
- * @param {*} options
117
- * @returns {AddedTuple}
118
- */
119
- addToQueue(keyScore, queryKey, orphanedTime, queryHandler, query, priority, options) {
120
- const queryQueueObj = {
121
- queueId: options.queueId,
122
- queryHandler,
123
- query,
124
- queryKey,
125
- stageQueryKey: options.stageQueryKey,
126
- priority,
127
- requestId: options.requestId,
128
- addedToQueueTime: new Date().getTime()
129
- };
130
- const key = this.redisHash(queryKey);
131
- if (!this.queryDef[key]) {
132
- this.queryDef[key] = queryQueueObj;
133
- }
134
- let added = 0;
135
- if (!this.toProcess[key] && !this.active[key]) {
136
- this.toProcess[key] = {
137
- order: keyScore,
138
- queueId: options.queueId,
139
- key
140
- };
141
- added = 1;
142
- }
143
- this.recent[key] = {
144
- order: orphanedTime,
145
- key,
146
- queueId: options.queueId,
147
- };
148
- if (this.getQueueEventsBus) {
149
- this.getQueueEventsBus().emit({
150
- event: 'addedToQueue',
151
- redisQueuePrefix: this.redisQueuePrefix,
152
- queryKey: this.redisHash(queryKey),
153
- payload: queryQueueObj
154
- });
155
- }
156
- return [
157
- added,
158
- queryQueueObj.queueId,
159
- Object.keys(this.toProcess).length,
160
- queryQueueObj.addedToQueueTime
161
- ];
162
- }
163
- getToProcessQueries() {
164
- return this.queueArrayAsTuple(this.toProcess);
165
- }
166
- getActiveQueries() {
167
- return this.queueArrayAsTuple(this.active);
168
- }
169
- async getQueryAndRemove(queryKey) {
170
- const key = this.redisHash(queryKey);
171
- const query = this.queryDef[key];
172
- delete this.active[key];
173
- delete this.heartBeat[key];
174
- delete this.toProcess[key];
175
- delete this.recent[key];
176
- delete this.queryDef[key];
177
- delete this.processingLocks[key];
178
- return [query];
179
- }
180
- async cancelQuery(queryKey) {
181
- const [query] = await this.getQueryAndRemove(queryKey);
182
- if (this.getQueueEventsBus) {
183
- this.getQueueEventsBus().emit({
184
- event: 'cancelQuery',
185
- redisQueuePrefix: this.redisQueuePrefix,
186
- queryKey: this.redisHash(queryKey),
187
- payload: query
188
- });
189
- }
190
- return query;
191
- }
192
- async setResultAndRemoveQuery(queryKey, executionResult, processingId) {
193
- const key = this.redisHash(queryKey);
194
- if (this.processingLocks[key] !== processingId) {
195
- return false;
196
- }
197
- const promise = this.getResultPromise(this.resultListKey(queryKey));
198
- delete this.active[key];
199
- delete this.heartBeat[key];
200
- delete this.toProcess[key];
201
- delete this.recent[key];
202
- delete this.queryDef[key];
203
- delete this.processingLocks[key];
204
- promise.resolved = true;
205
- promise.resolve(executionResult);
206
- if (this.getQueueEventsBus) {
207
- this.getQueueEventsBus().emit({
208
- event: 'setResultAndRemoveQuery',
209
- redisQueuePrefix: this.redisQueuePrefix,
210
- queryKey: this.redisHash(queryKey),
211
- payload: executionResult
212
- });
213
- }
214
- return true;
215
- }
216
- getNextProcessingId() {
217
- this.processingCounter.counter = this.processingCounter.counter ? this.processingCounter.counter + 1 : 1;
218
- return this.processingCounter.counter;
219
- }
220
- getOrphanedQueries() {
221
- return this.queueArrayAsTuple(this.recent, new Date().getTime());
222
- }
223
- getStalledQueries() {
224
- return this.queueArrayAsTuple(this.heartBeat, new Date().getTime() - this.heartBeatTimeout * 1000);
225
- }
226
- async getQueryStageState(onlyKeys) {
227
- return [this.queueArray(this.active), this.queueArray(this.toProcess), onlyKeys ? {} : ramda_1.default.clone(this.queryDef)];
228
- }
229
- async getQueryDef(queryKey) {
230
- return this.queryDef[queryKey];
231
- }
232
- /**
233
- * Updates heart beat for the processing query by its `queryKey`.
234
- *
235
- * @param {string} queryKey
236
- */
237
- updateHeartBeat(queryKey) {
238
- const key = this.redisHash(queryKey);
239
- if (this.heartBeat[key]) {
240
- this.heartBeat[key] = { key, order: new Date().getTime() };
241
- }
242
- }
243
- retrieveForProcessing(queryKey, processingId) {
244
- const key = this.redisHash(queryKey);
245
- let lockAcquired = false;
246
- if (!this.processingLocks[key]) {
247
- this.processingLocks[key] = processingId;
248
- lockAcquired = true;
249
- }
250
- else {
251
- return null;
252
- }
253
- let added = 0;
254
- if (Object.keys(this.active).length < this.concurrency && !this.active[key]) {
255
- this.active[key] = { key, order: processingId, queueId: processingId };
256
- delete this.toProcess[key];
257
- added = 1;
258
- }
259
- this.heartBeat[key] = { key, order: new Date().getTime() };
260
- if (this.getQueueEventsBus) {
261
- this.getQueueEventsBus().emit({
262
- event: 'retrievedForProcessing',
263
- redisQueuePrefix: this.redisQueuePrefix,
264
- queryKey: this.redisHash(queryKey),
265
- payload: this.queryDef[key]
266
- });
267
- }
268
- return [
269
- added,
270
- this.queryDef[key]?.queueId,
271
- this.queueArray(this.active),
272
- Object.keys(this.toProcess).length,
273
- this.queryDef[key],
274
- lockAcquired
275
- ];
276
- }
277
- freeProcessingLock(queryKey, processingId, activated) {
278
- const key = this.redisHash(queryKey);
279
- if (this.processingLocks[key] === processingId) {
280
- delete this.processingLocks[key];
281
- if (activated) {
282
- delete this.active[key];
283
- }
284
- }
285
- }
286
- async optimisticQueryUpdate(queryKey, toUpdate, processingId) {
287
- const key = this.redisHash(queryKey);
288
- if (this.processingLocks[key] !== processingId) {
289
- return false;
290
- }
291
- this.queryDef[key] = { ...this.queryDef[key], ...toUpdate };
292
- return true;
293
- }
294
- // eslint-disable-next-line @typescript-eslint/no-empty-function
295
- release() {
296
- }
297
- /**
298
- * Returns cache key to the specified by the queryKey query and the
299
- * specified by the suffix query state.
300
- * @param {*} queryKey
301
- * @param {string} suffix
302
- * @returns {string}
303
- */
304
- queryRedisKey(queryKey, suffix) {
305
- return `${this.redisQueuePrefix}_${this.redisHash(queryKey)}_${suffix}`;
306
- }
307
- /**
308
- * Returns cache key to the cached query result.
309
- * @param {*} queryKey
310
- * @returns {string}
311
- */
312
- resultListKey(queryKey) {
313
- return this.queryRedisKey(queryKey, 'RESULT');
314
- }
315
- /**
316
- * Returns hash sum of the query specified by the queryKey.
317
- * @param {*} queryKey
318
- * @returns {string}
319
- */
320
- redisHash(queryKey) {
321
- return this.driver.redisHash(queryKey);
5
+ const LocalQueueDriverConnection_1 = require("./LocalQueueDriverConnection");
6
+ const sharedState = {};
7
+ function getState(prefix) {
8
+ if (!sharedState[prefix]) {
9
+ sharedState[prefix] = new LocalQueueDriverConnection_1.LocalQueueDriverConnectionState();
322
10
  }
11
+ return sharedState[prefix];
323
12
  }
324
- exports.LocalQueueDriverConnection = LocalQueueDriverConnection;
325
- const results = {};
326
- const resultPromises = {};
327
- const queryDef = {};
328
- const toProcess = {};
329
- const recent = {};
330
- const active = {};
331
- const heartBeat = {};
332
- const processingCounters = {};
333
- const processingLocks = {};
334
- /**
335
- * @implements {QueueDriverInterface}
336
- */
337
13
  class LocalQueueDriver extends BaseQueueDriver_1.BaseQueueDriver {
14
+ options;
338
15
  constructor(options) {
339
- super(options.processUid);
16
+ super(options.processUid || 'local');
340
17
  this.options = options;
341
- results[options.redisQueuePrefix] = results[options.redisQueuePrefix] || {};
342
- resultPromises[options.redisQueuePrefix] = resultPromises[options.redisQueuePrefix] || {};
343
- queryDef[options.redisQueuePrefix] = queryDef[options.redisQueuePrefix] || {};
344
- toProcess[options.redisQueuePrefix] = toProcess[options.redisQueuePrefix] || {};
345
- recent[options.redisQueuePrefix] = recent[options.redisQueuePrefix] || {};
346
- active[options.redisQueuePrefix] = active[options.redisQueuePrefix] || {};
347
- heartBeat[options.redisQueuePrefix] = heartBeat[options.redisQueuePrefix] || {};
348
- processingCounters[options.redisQueuePrefix] = processingCounters[options.redisQueuePrefix] || {};
349
- processingLocks[options.redisQueuePrefix] = processingLocks[options.redisQueuePrefix] || {};
350
- this.results = results[options.redisQueuePrefix];
351
- this.resultPromises = resultPromises[options.redisQueuePrefix];
352
- this.queryDef = queryDef[options.redisQueuePrefix];
353
- this.toProcess = toProcess[options.redisQueuePrefix];
354
- this.recent = recent[options.redisQueuePrefix];
355
- this.active = active[options.redisQueuePrefix];
356
- this.heartBeat = heartBeat[options.redisQueuePrefix];
357
- this.processingCounter = processingCounters[options.redisQueuePrefix];
358
- this.processingLocks = processingLocks[options.redisQueuePrefix];
359
18
  }
360
- createConnection() {
361
- return new LocalQueueDriverConnection(this, this.options);
19
+ async createConnection() {
20
+ const state = getState(this.options.redisQueuePrefix);
21
+ return new LocalQueueDriverConnection_1.LocalQueueDriverConnection(this, state, this.options);
362
22
  }
363
23
  release(client) {
364
24
  client.release();
@@ -1 +1 @@
1
- {"version":3,"file":"LocalQueueDriver.js","sourceRoot":"","sources":["../../../src/orchestrator/LocalQueueDriver.js"],"names":[],"mappings":";;;;;;AAAA,kDAAsB;AACtB,6DAAmG;AACnG,uDAAoD;AAEpD;;GAEG;AACH,MAAa,0BAA0B;IACrC,YAAY,MAAM,EAAE,OAAO;QACzB,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;QAC9C,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC5C,IAAI,CAAC,iBAAiB,EAAE;YACxB,IAAI,CAAC,kBAAkB,EAAE;SAC1B,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEzD,OAAO;YACL,MAAM;YACN,SAAS;SACV,CAAC;IACJ,CAAC;IAED,gBAAgB,CAAC,aAAa;QAC5B,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE;YACvC,IAAI,aAAa,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;gBACzD,aAAa,GAAG,OAAO,CAAC;YAC1B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,OAAO,GAAG,aAAa,CAAC;SAC5D;QACD,OAAO,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,YAAY;QAClC,4CAA4C;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,EAAE;YACvE,OAAO,IAAI,CAAC;SACb;QACD,MAAM,cAAc,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAEvG,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;YACpC,cAAc,CAAC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;SAChD,CAAC,CAAC;QAEH,IAAI,GAAG,EAAE;YACP,OAAO,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;SAC3C;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,SAAS,CAAC,QAAQ;QACtB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;YACrF,OAAO,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;SACzC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,mBAAmB;QACtC,OAAO,eAAC,CAAC,IAAI,CACX,eAAC,CAAC,MAAM,EACR,eAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,eAAC,CAAC,QAAQ,CAAC,EAC/E,eAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EACtB,eAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAClB,CAAC,QAAQ,CAAC,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,iBAAiB,CAAC,QAAQ,EAAE,mBAAmB;QAC7C,OAAO,eAAC,CAAC,IAAI,CACX,eAAC,CAAC,MAAM,EACR,eAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,eAAC,CAAC,QAAQ,CAAC,EAC/E,eAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EACtB,eAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAC/B,CAAC,QAAQ,CAAC,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO;QACjF,MAAM,aAAa,GAAG;YACpB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,YAAY;YACZ,KAAK;YACL,QAAQ;YACR,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,QAAQ;YACR,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE;SACvC,CAAC;QAEF,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YACvB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC;SACpC;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG;gBACpB,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,GAAG;aACJ,CAAC;YAEF,KAAK,GAAG,CAAC,CAAC;SACX;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG;YACjB,KAAK,EAAE,YAAY;YACnB,GAAG;YACH,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,iBAAiB,EAAE,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,cAAc;gBACrB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBAClC,OAAO,EAAE,aAAa;aACvB,CAAC,CAAC;SACJ;QAED,OAAO;YACL,KAAK;YACL,aAAa,CAAC,OAAO;YACrB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM;YAClC,aAAa,CAAC,gBAAgB;SAC/B,CAAC;IACJ,CAAC;IAED,mBAAmB;QACjB,OAAO,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,QAAQ;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAQ;QACxB,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,iBAAiB,EAAE,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,aAAa;gBACpB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBAClC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;SACJ;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,QAAQ,EAAE,eAAe,EAAE,YAAY;QACnE,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,YAAY,EAAE;YAC9C,OAAO,KAAK,CAAC;SACd;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACjC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAEjC,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,iBAAiB,EAAE,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,yBAAyB;gBAChC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBAClC,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;SACJ;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,mBAAmB;QACjB,IAAI,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzG,OAAO,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC;IACxC,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IACrG,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,QAAQ;QAC/B,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IACjH,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAQ;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,QAAQ;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE;YACvB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;SAC5D;IACH,CAAC;IAED,qBAAqB,CAAC,QAAQ,EAAE,YAAY;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,YAAY,GAAG,KAAK,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE;YAC9B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;YACzC,YAAY,GAAG,IAAI,CAAC;SACrB;aAAM;YACL,OAAO,IAAI,CAAC;SACb;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC3E,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;YACvE,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAE3B,KAAK,GAAG,CAAC,CAAC;SACX;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAE3D,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,IAAI,CAAC,iBAAiB,EAAE,CAAC,IAAI,CAAC;gBAC5B,KAAK,EAAE,wBAAwB;gBAC/B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBAClC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;aAC5B,CAAC,CAAC;SACJ;QAED,OAAO;YACL,KAAK;YACL,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO;YAC3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM;YAClC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClB,YAAY;SACb,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,SAAS;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,YAAY,EAAE;YAC9C,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,SAAS,EAAE;gBACb,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;aACzB;SACF;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY;QAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,YAAY,EAAE;YAC9C,OAAO,KAAK,CAAC;SACd;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC5D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gEAAgE;IAChE,OAAO;IACP,CAAC;IAED;;;;;;OAMG;IACH,aAAa,CAAC,QAAQ,EAAE,MAAM;QAC5B,OAAO,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;IAC1E,CAAC;IAED;;;;OAIG;IACH,aAAa,CAAC,QAAQ;QACpB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,QAAQ;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;CACF;AAhXD,gEAgXC;AAED,MAAM,OAAO,GAAG,EAAE,CAAC;AACnB,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,QAAQ,GAAG,EAAE,CAAC;AACpB,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,MAAM,GAAG,EAAE,CAAC;AAClB,MAAM,MAAM,GAAG,EAAE,CAAC;AAClB,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B;;GAEG;AACH,MAAa,gBAAiB,SAAQ,iCAAe;IACnD,YAAY,OAAO;QACjB,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC5E,cAAc,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC1F,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC9E,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAChF,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC1E,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC1E,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAChF,kBAAkB,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAClG,eAAe,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAC5F,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACjD,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/D,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACrD,IAAI,CAAC,iBAAiB,GAAG,kBAAkB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACtE,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACnE,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,0BAA0B,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,MAAM;QACZ,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;CACF;AA/BD,4CA+BC"}
1
+ {"version":3,"file":"LocalQueueDriver.js","sourceRoot":"","sources":["../../../src/orchestrator/LocalQueueDriver.ts"],"names":[],"mappings":";;;AAIA,uDAAoD;AACpD,6EAA2G;AAE3G,MAAM,WAAW,GAAoD,EAAE,CAAC;AAExE,SAAS,QAAQ,CAAC,MAAc;IAC9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACxB,WAAW,CAAC,MAAM,CAAC,GAAG,IAAI,4DAA+B,EAAE,CAAC;KAC7D;IAED,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;AAC7B,CAAC;AAED,MAAa,gBAAiB,SAAQ,iCAAe;IAClC,OAAO,CAAqB;IAE7C,YAAmB,OAA2B;QAC5C,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAEM,KAAK,CAAC,gBAAgB;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACtD,OAAO,IAAI,uDAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACnE,CAAC;IAEM,OAAO,CAAC,MAAsC;QACnD,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;CACF;AAhBD,4CAgBC"}
@@ -0,0 +1,70 @@
1
+ import { QueueDriverConnectionInterface, QueryKey, QueryKeyHash, QueueId, ProcessingId, QueryDef, AddToQueueQuery, AddToQueueOptions, AddToQueueResponse, QueryKeysTuple, GetActiveAndToProcessResponse, QueryStageStateResponse, RetrieveForProcessingResponse, QueueDriverOptions } from '@cubejs-backend/base-driver';
2
+ import { LocalQueueDriver } from './LocalQueueDriver';
3
+ export interface QueueItem {
4
+ order: number;
5
+ key: QueryKeyHash;
6
+ queueId: QueueId;
7
+ }
8
+ export interface QueryDefObject {
9
+ queueId: QueueId;
10
+ queryHandler: string;
11
+ query: any;
12
+ queryKey: QueryKey;
13
+ stageQueryKey: string;
14
+ priority: number;
15
+ requestId: string;
16
+ addedToQueueTime: number;
17
+ }
18
+ export interface PromiseWithResolve<T = any> extends Promise<T> {
19
+ resolve?: (value: T) => void;
20
+ resolved?: boolean;
21
+ }
22
+ export interface ProcessingCounter {
23
+ counter: number;
24
+ }
25
+ export declare class LocalQueueDriverConnectionState {
26
+ resultPromises: Record<QueryKeyHash, PromiseWithResolve>;
27
+ queryDef: Record<QueryKeyHash, QueryDefObject>;
28
+ toProcess: Record<QueryKeyHash, QueueItem>;
29
+ recent: Record<QueryKeyHash, QueueItem>;
30
+ active: Record<QueryKeyHash, QueueItem>;
31
+ heartBeat: Record<QueryKeyHash, QueueItem>;
32
+ processingCounter: ProcessingCounter;
33
+ processingLocks: Record<QueryKeyHash, any>;
34
+ }
35
+ export declare class LocalQueueDriverConnection implements QueueDriverConnectionInterface {
36
+ private redisQueuePrefix;
37
+ private continueWaitTimeout;
38
+ private heartBeatTimeout;
39
+ private concurrency;
40
+ private driver;
41
+ private state;
42
+ constructor(driver: LocalQueueDriver, state: LocalQueueDriverConnectionState, options: QueueDriverOptions);
43
+ getQueriesToCancel(): Promise<QueryKeysTuple[]>;
44
+ getActiveAndToProcess(): Promise<GetActiveAndToProcessResponse>;
45
+ getResultPromise(resultListKey: string): PromiseWithResolve;
46
+ getResultBlocking(queryKeyHash: QueryKeyHash, _queueId?: QueueId): Promise<any>;
47
+ getResult(queryKey: QueryKey): Promise<any>;
48
+ protected queueArray(queueObj: Record<QueryKeyHash, QueueItem>, orderFilterLessThan?: number): string[];
49
+ protected queueArrayAsTuple(queueObj: Record<QueryKeyHash, QueueItem>, orderFilterLessThan?: number): QueryKeysTuple[];
50
+ addToQueue(keyScore: number, queryKey: QueryKey, orphanedTime: number, queryHandler: string, query: AddToQueueQuery, priority: number, options: AddToQueueOptions): Promise<AddToQueueResponse>;
51
+ getToProcessQueries(): Promise<QueryKeysTuple[]>;
52
+ getActiveQueries(): Promise<QueryKeysTuple[]>;
53
+ getQueryAndRemove(queryKeyHash: QueryKeyHash, _queueId?: QueueId | null): Promise<[QueryDef]>;
54
+ cancelQuery(queryKey: QueryKey, queueId?: QueueId | null): Promise<QueryDef | null>;
55
+ setResultAndRemoveQuery(queryKeyHash: QueryKeyHash, executionResult: any, processingId: ProcessingId, _queueId?: QueueId | null): Promise<boolean>;
56
+ getNextProcessingId(): Promise<ProcessingId>;
57
+ getOrphanedQueries(): Promise<QueryKeysTuple[]>;
58
+ getStalledQueries(): Promise<QueryKeysTuple[]>;
59
+ getQueryStageState(onlyKeys: boolean): Promise<QueryStageStateResponse>;
60
+ getQueryDef(queryKeyHash: QueryKeyHash, _queueId?: QueueId | null): Promise<QueryDef | null>;
61
+ updateHeartBeat(queryKeyHash: QueryKeyHash, queueId?: QueueId | null): Promise<void>;
62
+ retrieveForProcessing(queryKeyHash: QueryKeyHash, processingId: ProcessingId): Promise<RetrieveForProcessingResponse>;
63
+ freeProcessingLock(queryKeyHash: QueryKeyHash, processingId: ProcessingId, activated: any): Promise<void>;
64
+ optimisticQueryUpdate(queryKeyHash: QueryKeyHash, toUpdate: any, processingId: ProcessingId, _queueId?: QueueId | null): Promise<boolean>;
65
+ release(): void;
66
+ queryRedisKey(queryKey: QueryKey, suffix: string): string;
67
+ resultListKey(queryKey: QueryKey | QueryKeyHash): string;
68
+ redisHash(queryKey: QueryKey): QueryKeyHash;
69
+ }
70
+ //# sourceMappingURL=LocalQueueDriverConnection.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalQueueDriverConnection.d.ts","sourceRoot":"","sources":["../../../src/orchestrator/LocalQueueDriverConnection.ts"],"names":[],"mappings":"AACA,OAAO,EACL,8BAA8B,EAC9B,QAAQ,EACR,YAAY,EACZ,OAAO,EACP,YAAY,EACZ,QAAQ,EACR,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,6BAA6B,EAC7B,uBAAuB,EACvB,6BAA6B,EAC7B,kBAAkB,EACnB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,GAAG,CAAC;IACX,QAAQ,EAAE,QAAQ,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB,CAAC,CAAC,GAAG,GAAG,CAAE,SAAQ,OAAO,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,IAAI,CAAC;IAC7B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,+BAA+B;IACnC,cAAc,EAAE,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAM;IAE9D,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,CAAM;IAEpD,SAAS,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAM;IAEhD,MAAM,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAM;IAE7C,MAAM,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAM;IAE7C,SAAS,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,CAAM;IAEhD,iBAAiB,EAAE,iBAAiB,CAAkB;IAEtD,eAAe,EAAE,MAAM,CAAC,YAAY,EAAE,GAAG,CAAC,CAAM;CACxD;AAED,qBAAa,0BAA2B,YAAW,8BAA8B;IAC/E,OAAO,CAAC,gBAAgB,CAAS;IAEjC,OAAO,CAAC,mBAAmB,CAAS;IAEpC,OAAO,CAAC,gBAAgB,CAAS;IAEjC,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO,CAAC,MAAM,CAAmB;IAEjC,OAAO,CAAC,KAAK,CAAkC;gBAE5B,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,+BAA+B,EAAE,OAAO,EAAE,kBAAkB;IASnG,kBAAkB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAS/C,qBAAqB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAUrE,gBAAgB,CAAC,aAAa,EAAE,MAAM,GAAG,kBAAkB;IAYrD,iBAAiB,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC;IAkB/E,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IASxD,SAAS,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,mBAAmB,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IASvG,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,mBAAmB,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE;IASzG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA4C/L,mBAAmB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAIhD,gBAAgB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAI7C,iBAAiB,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC;IAa7F,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAKnF,uBAAuB,CAAC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,GAAG,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;IAsBlJ,mBAAmB,IAAI,OAAO,CAAC,YAAY,CAAC;IAK5C,kBAAkB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAI/C,iBAAiB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAI9C,kBAAkB,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAIvE,WAAW,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAI5F,eAAe,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAMpF,qBAAqB,CAAC,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,6BAA6B,CAAC;IA+BrH,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IASzG,qBAAqB,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;IAS/I,OAAO,IAAI,IAAI;IAIf,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAIzD,aAAa,CAAC,QAAQ,EAAE,QAAQ,GAAG,YAAY,GAAG,MAAM;IAIxD,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,YAAY;CAGnD"}