@cubejs-backend/query-orchestrator 1.7.24 → 1.7.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/orchestrator/LocalQueueDriverConnection.d.ts +3 -2
- package/dist/src/orchestrator/LocalQueueDriverConnection.d.ts.map +1 -1
- package/dist/src/orchestrator/LocalQueueDriverConnection.js +11 -5
- package/dist/src/orchestrator/LocalQueueDriverConnection.js.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoadCache.d.ts +2 -2
- package/dist/src/orchestrator/PreAggregationLoadCache.d.ts.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoadCache.js +1 -1
- package/dist/src/orchestrator/PreAggregationLoadCache.js.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoader.d.ts +3 -3
- package/dist/src/orchestrator/PreAggregationLoader.d.ts.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoader.js +16 -16
- package/dist/src/orchestrator/PreAggregationLoader.js.map +1 -1
- package/dist/src/orchestrator/PreAggregationPartitionRangeLoader.js +3 -2
- package/dist/src/orchestrator/PreAggregationPartitionRangeLoader.js.map +1 -1
- package/dist/src/orchestrator/QueryCache.d.ts +4 -7
- package/dist/src/orchestrator/QueryCache.d.ts.map +1 -1
- package/dist/src/orchestrator/QueryCache.js +18 -22
- package/dist/src/orchestrator/QueryCache.js.map +1 -1
- package/dist/src/orchestrator/QueryQueue.d.ts +33 -5
- package/dist/src/orchestrator/QueryQueue.d.ts.map +1 -1
- package/dist/src/orchestrator/QueryQueue.js +326 -238
- package/dist/src/orchestrator/QueryQueue.js.map +1 -1
- package/package.json +6 -6
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.QueryQueue = void 0;
|
|
4
4
|
const events_1 = require("events");
|
|
5
5
|
const shared_1 = require("@cubejs-backend/shared");
|
|
6
|
+
const base_driver_1 = require("@cubejs-backend/base-driver");
|
|
6
7
|
const cubestore_driver_1 = require("@cubejs-backend/cubestore-driver");
|
|
7
8
|
const TimeoutError_1 = require("./TimeoutError");
|
|
8
9
|
const ContinueWaitError_1 = require("./ContinueWaitError");
|
|
@@ -53,7 +54,7 @@ class QueryQueue {
|
|
|
53
54
|
this.executionTimeout = options.executionTimeout || (0, shared_1.getEnv)('dbQueryTimeout');
|
|
54
55
|
this.orphanedTimeout = options.orphanedTimeout || 120;
|
|
55
56
|
this.heartBeatInterval = options.heartBeatInterval || 30;
|
|
56
|
-
this.sendProcessMessageFn = options.sendProcessMessageFn || ((
|
|
57
|
+
this.sendProcessMessageFn = options.sendProcessMessageFn || ((retrieved) => { this.executeQuery(retrieved); });
|
|
57
58
|
this.sendCancelMessageFn = options.sendCancelMessageFn || ((query, queueId) => { this.processCancel(query, queueId); });
|
|
58
59
|
this.queryHandlers = options.queryHandlers;
|
|
59
60
|
this.streamHandler = options.streamHandler;
|
|
@@ -107,7 +108,7 @@ class QueryQueue {
|
|
|
107
108
|
*
|
|
108
109
|
* @throw {ContinueWaitError}
|
|
109
110
|
*/
|
|
110
|
-
async executeInQueue(queryHandler, queryKey, query, priority, executeOptions) {
|
|
111
|
+
async executeInQueue(queryHandler, queryKey, query, priority = base_driver_1.QueuePriority.Background, executeOptions) {
|
|
111
112
|
const options = {
|
|
112
113
|
queueId: this.generateQueueId(),
|
|
113
114
|
...executeOptions,
|
|
@@ -143,10 +144,8 @@ class QueryQueue {
|
|
|
143
144
|
}
|
|
144
145
|
const queueConnection = await this.queueDriver.createConnection();
|
|
145
146
|
let waitingContext;
|
|
147
|
+
let streamWait = null;
|
|
146
148
|
try {
|
|
147
|
-
if (priority == null) {
|
|
148
|
-
priority = 0;
|
|
149
|
-
}
|
|
150
149
|
if (!(priority >= -10000 && priority <= 10000)) {
|
|
151
150
|
throw new Error('Priority should be between -10000 and 10000');
|
|
152
151
|
}
|
|
@@ -163,12 +162,8 @@ class QueryQueue {
|
|
|
163
162
|
if (jobExists)
|
|
164
163
|
return null;
|
|
165
164
|
}
|
|
166
|
-
const time = new Date().getTime();
|
|
167
|
-
const keyScore = time + (10000 - priority) * 1E14;
|
|
168
165
|
options.orphanedTimeout = query.orphanedTimeout;
|
|
169
|
-
const
|
|
170
|
-
const orphanedTime = time + (orphanedTimeout * 1000);
|
|
171
|
-
const [added, queueId, queueSize, addedToQueueTime] = await queueConnection.addToQueue(keyScore, queryKey, orphanedTime, queryHandler, query, priority, options);
|
|
166
|
+
const [added, queueId, queueSize, addedToQueueTime, retrieved] = await queueConnection.addToQueue(queryKey, queryHandler, query, priority, options);
|
|
172
167
|
if (added > 0) {
|
|
173
168
|
waitingContext = {
|
|
174
169
|
queueId,
|
|
@@ -193,11 +188,23 @@ class QueryQueue {
|
|
|
193
188
|
preAggregation: query.preAggregation,
|
|
194
189
|
addedToQueueTime,
|
|
195
190
|
persistent: !!queryKey.persistent,
|
|
191
|
+
fastTrack: !!retrieved,
|
|
196
192
|
});
|
|
197
193
|
}
|
|
198
|
-
|
|
194
|
+
// Subscribing after the dispatch would lose the `streamStarted` event of a handler
|
|
195
|
+
// which starts fast
|
|
196
|
+
if (queryHandler === 'stream') {
|
|
197
|
+
streamWait = this.waitForQueryStream(queryKeyHash);
|
|
198
|
+
}
|
|
199
|
+
if (retrieved) {
|
|
200
|
+
// The item is active already, there is nothing for reconcile to pick up
|
|
201
|
+
await this.dispatchQuery(this.retrievedQuery(queryKeyHash, queueId, retrieved));
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
await this.reconcileQueue();
|
|
205
|
+
}
|
|
199
206
|
if (!added) {
|
|
200
|
-
const queryDef = await queueConnection.getQueryDef(queryKeyHash, queueId);
|
|
207
|
+
const queryDef = retrieved ? retrieved[4] : await queueConnection.getQueryDef(queryKeyHash, queueId);
|
|
201
208
|
if (queryDef) {
|
|
202
209
|
waitingContext = {
|
|
203
210
|
queueId,
|
|
@@ -209,44 +216,20 @@ class QueryQueue {
|
|
|
209
216
|
};
|
|
210
217
|
}
|
|
211
218
|
}
|
|
212
|
-
|
|
219
|
+
// A retrieval carries the active keys of its prefix, and a retrieved query is never pending,
|
|
220
|
+
// so it has no place in the queue to report
|
|
221
|
+
const [active, toProcess] = retrieved ? [retrieved[2], undefined] : await queueConnection.getQueryStageState(true);
|
|
213
222
|
this.logger('Waiting for query', {
|
|
214
223
|
...waitingContext,
|
|
215
224
|
queueSize,
|
|
216
225
|
activeQueryKeys: active,
|
|
217
226
|
toProcessQueryKeys: toProcess,
|
|
218
227
|
active: active.indexOf(queryKeyHash) !== -1,
|
|
219
|
-
queueIndex: toProcess.indexOf(queryKeyHash),
|
|
228
|
+
queueIndex: toProcess ? toProcess.indexOf(queryKeyHash) : -1,
|
|
229
|
+
fastTrack: !!retrieved,
|
|
220
230
|
});
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
// TODO: Fix an issue with a fast execution of stream handler which caused by removal of QueryStream from streams,
|
|
224
|
-
// while EventListener doesnt start to listen for started stream event
|
|
225
|
-
if (queryHandler === 'stream') {
|
|
226
|
-
const self = this;
|
|
227
|
-
result = await new Promise((resolve) => {
|
|
228
|
-
let timeoutTimerId = null;
|
|
229
|
-
const onStreamStarted = (streamStartedHash) => {
|
|
230
|
-
if (streamStartedHash === queryKeyHash) {
|
|
231
|
-
if (timeoutTimerId) {
|
|
232
|
-
clearTimeout(timeoutTimerId);
|
|
233
|
-
}
|
|
234
|
-
resolve(self.getQueryStream(queryKeyHash));
|
|
235
|
-
}
|
|
236
|
-
};
|
|
237
|
-
self.streamEvents.addListener('streamStarted', onStreamStarted);
|
|
238
|
-
const stream = this.getQueryStream(queryKeyHash);
|
|
239
|
-
if (stream) {
|
|
240
|
-
self.streamEvents.removeListener('streamStarted', onStreamStarted);
|
|
241
|
-
resolve(stream);
|
|
242
|
-
}
|
|
243
|
-
else {
|
|
244
|
-
timeoutTimerId = setTimeout(() => {
|
|
245
|
-
self.streamEvents.removeListener('streamStarted', onStreamStarted);
|
|
246
|
-
resolve(null);
|
|
247
|
-
}, this.continueWaitTimeout * 10000);
|
|
248
|
-
}
|
|
249
|
-
});
|
|
231
|
+
if (streamWait) {
|
|
232
|
+
result = await streamWait.promise;
|
|
250
233
|
}
|
|
251
234
|
else {
|
|
252
235
|
// Result here won't be fetched for a jobed build query (initialized by
|
|
@@ -266,9 +249,58 @@ class QueryQueue {
|
|
|
266
249
|
throw error;
|
|
267
250
|
}
|
|
268
251
|
finally {
|
|
252
|
+
streamWait?.dispose();
|
|
269
253
|
this.queueDriver.release(queueConnection);
|
|
270
254
|
}
|
|
271
255
|
}
|
|
256
|
+
retrievedQuery(queryKeyHash, queueId, retrieved) {
|
|
257
|
+
const [, , , queueSize, query] = retrieved;
|
|
258
|
+
return {
|
|
259
|
+
queryKeyHash,
|
|
260
|
+
queueId,
|
|
261
|
+
processingId: queueId,
|
|
262
|
+
queueSize,
|
|
263
|
+
query,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* `dispose` releases the listener and the timer, it's a no-op once the promise resolved.
|
|
268
|
+
*/
|
|
269
|
+
waitForQueryStream(queryKeyHash) {
|
|
270
|
+
let timeoutTimerId = null;
|
|
271
|
+
let onStreamStarted = null;
|
|
272
|
+
const dispose = () => {
|
|
273
|
+
if (timeoutTimerId) {
|
|
274
|
+
clearTimeout(timeoutTimerId);
|
|
275
|
+
timeoutTimerId = null;
|
|
276
|
+
}
|
|
277
|
+
if (onStreamStarted) {
|
|
278
|
+
this.streamEvents.removeListener('streamStarted', onStreamStarted);
|
|
279
|
+
onStreamStarted = null;
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
const promise = new Promise((resolve) => {
|
|
283
|
+
onStreamStarted = (streamStartedHash) => {
|
|
284
|
+
if (streamStartedHash === queryKeyHash) {
|
|
285
|
+
dispose();
|
|
286
|
+
resolve(this.getQueryStream(queryKeyHash) ?? null);
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
this.streamEvents.addListener('streamStarted', onStreamStarted);
|
|
290
|
+
const stream = this.getQueryStream(queryKeyHash);
|
|
291
|
+
if (stream) {
|
|
292
|
+
dispose();
|
|
293
|
+
resolve(stream);
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
timeoutTimerId = setTimeout(() => {
|
|
297
|
+
dispose();
|
|
298
|
+
resolve(null);
|
|
299
|
+
}, this.continueWaitTimeout * 10000);
|
|
300
|
+
}
|
|
301
|
+
});
|
|
302
|
+
return { promise, dispose };
|
|
303
|
+
}
|
|
272
304
|
/**
|
|
273
305
|
* @throw {Error}
|
|
274
306
|
*/
|
|
@@ -461,7 +493,8 @@ class QueryQueue {
|
|
|
461
493
|
}
|
|
462
494
|
})
|
|
463
495
|
.slice(0, toProcessLimit)
|
|
464
|
-
.map(([queryKey, queueId]) => this.
|
|
496
|
+
.map(([queryKey, queueId]) => this.processQuery(queryKey, queueId));
|
|
497
|
+
// Awaits the retrieval of every picked query, not their execution.
|
|
465
498
|
await Promise.all(tasks);
|
|
466
499
|
}
|
|
467
500
|
finally {
|
|
@@ -592,10 +625,35 @@ class QueryQueue {
|
|
|
592
625
|
return executionResult;
|
|
593
626
|
}
|
|
594
627
|
/**
|
|
595
|
-
*
|
|
596
|
-
* of the logic related to the queue updates, heartbeat, etc.
|
|
628
|
+
* Retrieves the query specified by the `queryKeyHashed` and hands it over for execution.
|
|
597
629
|
*/
|
|
598
630
|
async processQuery(queryKeyHashed, queueId) {
|
|
631
|
+
const retrieved = await this.retrieveQueryForProcessing(queryKeyHashed, queueId);
|
|
632
|
+
if (!retrieved) {
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
await this.dispatchQuery(retrieved);
|
|
636
|
+
}
|
|
637
|
+
async dispatchQuery(retrieved) {
|
|
638
|
+
try {
|
|
639
|
+
await this.sendProcessMessageFn(retrieved);
|
|
640
|
+
}
|
|
641
|
+
catch (e) {
|
|
642
|
+
this.logger('Error while processing message', {
|
|
643
|
+
queueId: retrieved.queueId,
|
|
644
|
+
queryKey: retrieved.query.queryKey,
|
|
645
|
+
requestId: retrieved.query.requestId,
|
|
646
|
+
error: (e.stack || e).toString(),
|
|
647
|
+
queuePrefix: this.redisQueuePrefix
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* Acquires the processing lock for the query specified by the `queryKeyHashed` and moves it to
|
|
653
|
+
* the active set. Returns `null` when the retrieval didn't succeed, which means another node is
|
|
654
|
+
* already running the query or the concurrency budget is full.
|
|
655
|
+
*/
|
|
656
|
+
async retrieveQueryForProcessing(queryKeyHashed, queueId) {
|
|
599
657
|
const queueConnection = await this.queueDriver.createConnection();
|
|
600
658
|
let insertedCount;
|
|
601
659
|
let activeKeys;
|
|
@@ -617,231 +675,261 @@ class QueryQueue {
|
|
|
617
675
|
if (!query) {
|
|
618
676
|
query = await queueConnection.getQueryDef(queryKeyHashed, null);
|
|
619
677
|
}
|
|
620
|
-
if (query
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
//
|
|
624
|
-
//
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
678
|
+
if (!query || !insertedCount || !activated || !processingLockAcquired) {
|
|
679
|
+
// TODO Ideally streaming queries should reconcile queue here after waiting on open slot however in practice continue wait timeout reconciles faster CPU-wise
|
|
680
|
+
// if (query?.queryHandler === 'stream') {
|
|
681
|
+
// const [active] = await queueConnection.getQueryStageState(true);
|
|
682
|
+
// if (active && active.length > 0) {
|
|
683
|
+
// await Promise.race(active.map(keyHash => queueConnection.getResultBlocking(keyHash)));
|
|
684
|
+
// await this.reconcileQueue();
|
|
685
|
+
// }
|
|
686
|
+
// }
|
|
687
|
+
this.logger('Skip processing', {
|
|
629
688
|
queueId,
|
|
630
689
|
processingId,
|
|
631
|
-
|
|
632
|
-
|
|
690
|
+
queryKey: query && query.queryKey || queryKeyHashed,
|
|
691
|
+
requestId: query && query.requestId,
|
|
633
692
|
queuePrefix: this.redisQueuePrefix,
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
693
|
+
processingLockAcquired,
|
|
694
|
+
query,
|
|
695
|
+
insertedCount,
|
|
696
|
+
activeKeys,
|
|
697
|
+
activated,
|
|
698
|
+
queryExists: !!query
|
|
641
699
|
});
|
|
642
|
-
await queueConnection.
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
await this.streamHandler(query.query, queryStream);
|
|
703
|
-
// CubeStore has special handling for null
|
|
704
|
-
executionResult = {
|
|
705
|
-
streamResult: true
|
|
706
|
-
};
|
|
707
|
-
}
|
|
708
|
-
finally {
|
|
709
|
-
if (this.streams.get(queryKeyHashed) === queryStream) {
|
|
710
|
-
this.streams.delete(queryKeyHashed);
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
|
-
break;
|
|
714
|
-
default:
|
|
715
|
-
executionResult = {
|
|
716
|
-
result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
|
|
717
|
-
localCancelHandler = cancelHandler;
|
|
718
|
-
try {
|
|
719
|
-
await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, processingId, queueId);
|
|
720
|
-
}
|
|
721
|
-
catch (e) {
|
|
722
|
-
this.logger('Error while query update', {
|
|
723
|
-
queueId,
|
|
724
|
-
queryKey: query.queryKey,
|
|
725
|
-
error: e.stack || e,
|
|
726
|
-
queuePrefix: this.redisQueuePrefix,
|
|
727
|
-
requestId: query.requestId,
|
|
728
|
-
metadata: query.query?.metadata,
|
|
729
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
730
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
731
|
-
preAggregation: query.query?.preAggregation,
|
|
732
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
733
|
-
});
|
|
734
|
-
}
|
|
735
|
-
}))
|
|
736
|
-
};
|
|
737
|
-
break;
|
|
738
|
-
}
|
|
739
|
-
this.logger('Performing query completed', {
|
|
700
|
+
await queueConnection.freeProcessingLock(queryKeyHashed, processingId, activated);
|
|
701
|
+
return null;
|
|
702
|
+
}
|
|
703
|
+
return {
|
|
704
|
+
queryKeyHash: queryKeyHashed,
|
|
705
|
+
queueId,
|
|
706
|
+
processingId,
|
|
707
|
+
queueSize,
|
|
708
|
+
query,
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
catch (e) {
|
|
712
|
+
this.logger('Queue storage error', {
|
|
713
|
+
queueId,
|
|
714
|
+
queryKey: query && query.queryKey || queryKeyHashed,
|
|
715
|
+
requestId: query && query.requestId,
|
|
716
|
+
error: (e.stack || e).toString(),
|
|
717
|
+
queuePrefix: this.redisQueuePrefix
|
|
718
|
+
});
|
|
719
|
+
return null;
|
|
720
|
+
}
|
|
721
|
+
finally {
|
|
722
|
+
this.queueDriver.release(queueConnection);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
/**
|
|
726
|
+
* Executes a retrieved query: runs its handler while keeping the queue heartbeat alive, then acks
|
|
727
|
+
* the result. It's the counterpart of `sendProcessMessageFn` and the entry point for a custom
|
|
728
|
+
* implementation which hands the query over to another process.
|
|
729
|
+
*/
|
|
730
|
+
async executeQuery(retrieved) {
|
|
731
|
+
const { queryKeyHash: queryKeyHashed, queueId, processingId, queueSize, query } = retrieved;
|
|
732
|
+
const queueConnection = await this.queueDriver.createConnection();
|
|
733
|
+
try {
|
|
734
|
+
let executionResult;
|
|
735
|
+
let queryExecutionFinished = false;
|
|
736
|
+
// Set by the query handler's setCancelHandler callback once execution begins.
|
|
737
|
+
// Not available on the original query def from retrieveForProcessing.
|
|
738
|
+
let localCancelHandler = null;
|
|
739
|
+
const startQueryTime = (new Date()).getTime();
|
|
740
|
+
const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
|
|
741
|
+
this.logger('Performing query', {
|
|
742
|
+
queueId,
|
|
743
|
+
processingId,
|
|
744
|
+
queueSize,
|
|
745
|
+
queryKey: query.queryKey,
|
|
746
|
+
queuePrefix: this.redisQueuePrefix,
|
|
747
|
+
requestId: query.requestId,
|
|
748
|
+
timeInQueue,
|
|
749
|
+
metadata: query.query?.metadata,
|
|
750
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
751
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
752
|
+
preAggregation: query.query?.preAggregation,
|
|
753
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
754
|
+
});
|
|
755
|
+
await queueConnection.optimisticQueryUpdate(queryKeyHashed, { startQueryTime }, processingId, queueId);
|
|
756
|
+
let queryProcessHeartbeat = Date.now();
|
|
757
|
+
const heartBeatTimer = setInterval(async () => {
|
|
758
|
+
if ((Date.now() - queryProcessHeartbeat) > 5 * 60 * 1000) {
|
|
759
|
+
this.logger('Query processing heartbeat', {
|
|
740
760
|
queueId,
|
|
741
|
-
processingId,
|
|
742
|
-
queueSize,
|
|
743
|
-
duration: ((new Date()).getTime() - startQueryTime),
|
|
744
761
|
queryKey: query.queryKey,
|
|
745
|
-
queuePrefix: this.redisQueuePrefix,
|
|
746
762
|
requestId: query.requestId,
|
|
747
|
-
timeInQueue,
|
|
748
|
-
metadata: query.query?.metadata,
|
|
749
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
750
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
751
|
-
preAggregation: query.query?.preAggregation,
|
|
752
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
753
763
|
});
|
|
764
|
+
queryProcessHeartbeat = Date.now();
|
|
765
|
+
}
|
|
766
|
+
try {
|
|
767
|
+
await queueConnection.updateHeartBeat(queryKeyHashed, queueId);
|
|
754
768
|
}
|
|
755
769
|
catch (e) {
|
|
756
|
-
|
|
757
|
-
error: (e.message || e).toString() // TODO error handling
|
|
758
|
-
};
|
|
759
|
-
this.logger('Error while querying', {
|
|
770
|
+
this.logger('Error updating heartbeat', {
|
|
760
771
|
queueId,
|
|
761
|
-
processingId,
|
|
762
|
-
queueSize,
|
|
763
|
-
duration: ((new Date()).getTime() - startQueryTime),
|
|
764
772
|
queryKey: query.queryKey,
|
|
773
|
+
error: e.stack || e,
|
|
765
774
|
queuePrefix: this.redisQueuePrefix,
|
|
766
775
|
requestId: query.requestId,
|
|
767
|
-
timeInQueue,
|
|
768
|
-
metadata: query.query?.metadata,
|
|
769
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
770
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
771
|
-
preAggregation: query.query?.preAggregation,
|
|
772
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
773
|
-
error: (e.stack || e).toString()
|
|
774
776
|
});
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
777
|
+
}
|
|
778
|
+
if (!queryExecutionFinished && localCancelHandler !== null) {
|
|
779
|
+
try {
|
|
780
|
+
const currentDef = await queueConnection.getQueryDef(queryKeyHashed, queueId);
|
|
781
|
+
if (!currentDef && !queryExecutionFinished) {
|
|
782
|
+
this.logger('Cancelling query due to external cancellation', {
|
|
779
783
|
queueId,
|
|
780
|
-
|
|
781
|
-
queryKey: queryWithCancelHandle.queryKey,
|
|
784
|
+
queryKey: query.queryKey,
|
|
782
785
|
queuePrefix: this.redisQueuePrefix,
|
|
783
|
-
requestId:
|
|
784
|
-
metadata:
|
|
785
|
-
preAggregationId:
|
|
786
|
-
newVersionEntry:
|
|
787
|
-
preAggregation:
|
|
788
|
-
addedToQueueTime:
|
|
786
|
+
requestId: query.requestId,
|
|
787
|
+
metadata: query.query?.metadata,
|
|
788
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
789
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
790
|
+
preAggregation: query.query?.preAggregation,
|
|
791
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
789
792
|
});
|
|
790
|
-
|
|
793
|
+
const cancelQuery = { ...query, cancelHandler: localCancelHandler };
|
|
794
|
+
await this.processCancel(cancelQuery, queueId);
|
|
791
795
|
}
|
|
792
796
|
}
|
|
797
|
+
catch (e) {
|
|
798
|
+
this.logger('Error checking for external cancellation', {
|
|
799
|
+
queueId,
|
|
800
|
+
queryKey: query.queryKey,
|
|
801
|
+
error: e.stack || e,
|
|
802
|
+
queuePrefix: this.redisQueuePrefix,
|
|
803
|
+
requestId: query.requestId,
|
|
804
|
+
});
|
|
805
|
+
}
|
|
793
806
|
}
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
807
|
+
}, this.heartBeatInterval * 1000);
|
|
808
|
+
try {
|
|
809
|
+
const handler = query?.queryHandler;
|
|
810
|
+
switch (handler) {
|
|
811
|
+
case 'stream':
|
|
812
|
+
// eslint-disable-next-line no-case-declarations
|
|
813
|
+
const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
|
|
814
|
+
try {
|
|
815
|
+
await this.streamHandler(query.query, queryStream);
|
|
816
|
+
// CubeStore has special handling for null
|
|
817
|
+
executionResult = {
|
|
818
|
+
streamResult: true
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
finally {
|
|
822
|
+
if (this.streams.get(queryKeyHashed) === queryStream) {
|
|
823
|
+
this.streams.delete(queryKeyHashed);
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
break;
|
|
827
|
+
default:
|
|
828
|
+
executionResult = {
|
|
829
|
+
result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
|
|
830
|
+
localCancelHandler = cancelHandler;
|
|
831
|
+
try {
|
|
832
|
+
await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, processingId, queueId);
|
|
833
|
+
}
|
|
834
|
+
catch (e) {
|
|
835
|
+
this.logger('Error while query update', {
|
|
836
|
+
queueId,
|
|
837
|
+
queryKey: query.queryKey,
|
|
838
|
+
error: e.stack || e,
|
|
839
|
+
queuePrefix: this.redisQueuePrefix,
|
|
840
|
+
requestId: query.requestId,
|
|
841
|
+
metadata: query.query?.metadata,
|
|
842
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
843
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
844
|
+
preAggregation: query.query?.preAggregation,
|
|
845
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
}))
|
|
849
|
+
};
|
|
850
|
+
break;
|
|
797
851
|
}
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
852
|
+
this.logger('Performing query completed', {
|
|
853
|
+
queueId,
|
|
854
|
+
processingId,
|
|
855
|
+
queueSize,
|
|
856
|
+
duration: ((new Date()).getTime() - startQueryTime),
|
|
857
|
+
queryKey: query.queryKey,
|
|
858
|
+
queuePrefix: this.redisQueuePrefix,
|
|
859
|
+
requestId: query.requestId,
|
|
860
|
+
timeInQueue,
|
|
861
|
+
metadata: query.query?.metadata,
|
|
862
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
863
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
864
|
+
preAggregation: query.query?.preAggregation,
|
|
865
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
catch (e) {
|
|
869
|
+
executionResult = {
|
|
870
|
+
error: (e.message || e).toString() // TODO error handling
|
|
871
|
+
};
|
|
872
|
+
this.logger('Error while querying', {
|
|
873
|
+
queueId,
|
|
874
|
+
processingId,
|
|
875
|
+
queueSize,
|
|
876
|
+
duration: ((new Date()).getTime() - startQueryTime),
|
|
877
|
+
queryKey: query.queryKey,
|
|
878
|
+
queuePrefix: this.redisQueuePrefix,
|
|
879
|
+
requestId: query.requestId,
|
|
880
|
+
timeInQueue,
|
|
881
|
+
metadata: query.query?.metadata,
|
|
882
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
883
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
884
|
+
preAggregation: query.query?.preAggregation,
|
|
885
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
886
|
+
error: (e.stack || e).toString()
|
|
887
|
+
});
|
|
888
|
+
if (e instanceof TimeoutError_1.TimeoutError) {
|
|
889
|
+
const queryWithCancelHandle = await queueConnection.getQueryDef(queryKeyHashed, queueId);
|
|
890
|
+
if (queryWithCancelHandle) {
|
|
891
|
+
this.logger('Cancelling query due to timeout', {
|
|
892
|
+
queueId,
|
|
893
|
+
processingId,
|
|
894
|
+
queryKey: queryWithCancelHandle.queryKey,
|
|
895
|
+
queuePrefix: this.redisQueuePrefix,
|
|
896
|
+
requestId: queryWithCancelHandle.requestId,
|
|
897
|
+
metadata: queryWithCancelHandle.query?.metadata,
|
|
898
|
+
preAggregationId: queryWithCancelHandle.query?.preAggregation?.preAggregationId,
|
|
899
|
+
newVersionEntry: queryWithCancelHandle.query?.newVersionEntry,
|
|
900
|
+
preAggregation: queryWithCancelHandle.query?.preAggregation,
|
|
901
|
+
addedToQueueTime: queryWithCancelHandle.addedToQueueTime,
|
|
902
|
+
});
|
|
903
|
+
await this.sendCancelMessageFn(queryWithCancelHandle, queueId);
|
|
904
|
+
}
|
|
812
905
|
}
|
|
813
|
-
await this.reconcileQueue();
|
|
814
906
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
// await this.reconcileQueue();
|
|
822
|
-
// }
|
|
823
|
-
// }
|
|
824
|
-
this.logger('Skip processing', {
|
|
907
|
+
finally {
|
|
908
|
+
queryExecutionFinished = true;
|
|
909
|
+
clearInterval(heartBeatTimer);
|
|
910
|
+
}
|
|
911
|
+
if (!(await queueConnection.setResultAndRemoveQuery(queryKeyHashed, executionResult, processingId, queueId))) {
|
|
912
|
+
this.logger('Orphaned execution result', {
|
|
825
913
|
queueId,
|
|
826
914
|
processingId,
|
|
827
|
-
|
|
828
|
-
|
|
915
|
+
warn: 'Result for query was not set due to processing lock wasn\'t acquired',
|
|
916
|
+
queryKey: query.queryKey,
|
|
829
917
|
queuePrefix: this.redisQueuePrefix,
|
|
830
|
-
|
|
831
|
-
query,
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
918
|
+
requestId: query.requestId,
|
|
919
|
+
metadata: query.query?.metadata,
|
|
920
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
921
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
922
|
+
preAggregation: query.query?.preAggregation,
|
|
923
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
836
924
|
});
|
|
837
|
-
await queueConnection.freeProcessingLock(queryKeyHashed, processingId, activated);
|
|
838
925
|
}
|
|
926
|
+
await this.reconcileQueue();
|
|
839
927
|
}
|
|
840
928
|
catch (e) {
|
|
841
929
|
this.logger('Queue storage error', {
|
|
842
930
|
queueId,
|
|
843
|
-
queryKey: query
|
|
844
|
-
requestId: query
|
|
931
|
+
queryKey: query.queryKey,
|
|
932
|
+
requestId: query.requestId,
|
|
845
933
|
error: (e.stack || e).toString(),
|
|
846
934
|
queuePrefix: this.redisQueuePrefix
|
|
847
935
|
});
|