@cubejs-backend/query-orchestrator 1.7.25 → 1.7.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/orchestrator/LocalQueueDriverConnection.d.ts +6 -12
- package/dist/src/orchestrator/LocalQueueDriverConnection.d.ts.map +1 -1
- package/dist/src/orchestrator/LocalQueueDriverConnection.js +31 -47
- package/dist/src/orchestrator/LocalQueueDriverConnection.js.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoadCache.d.ts +2 -2
- package/dist/src/orchestrator/PreAggregationLoadCache.d.ts.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoadCache.js.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoader.d.ts +3 -3
- package/dist/src/orchestrator/PreAggregationLoader.d.ts.map +1 -1
- package/dist/src/orchestrator/PreAggregationLoader.js +8 -8
- package/dist/src/orchestrator/PreAggregationLoader.js.map +1 -1
- package/dist/src/orchestrator/PreAggregationPartitionRangeLoader.js +3 -2
- package/dist/src/orchestrator/PreAggregationPartitionRangeLoader.js.map +1 -1
- package/dist/src/orchestrator/QueryCache.d.ts.map +1 -1
- package/dist/src/orchestrator/QueryCache.js +2 -1
- package/dist/src/orchestrator/QueryCache.js.map +1 -1
- package/dist/src/orchestrator/QueryQueue.d.ts +25 -6
- package/dist/src/orchestrator/QueryQueue.d.ts.map +1 -1
- package/dist/src/orchestrator/QueryQueue.js +292 -261
- package/dist/src/orchestrator/QueryQueue.js.map +1 -1
- package/package.json +10 -7
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.QueryQueue = void 0;
|
|
4
4
|
const events_1 = require("events");
|
|
5
5
|
const shared_1 = require("@cubejs-backend/shared");
|
|
6
|
+
const base_driver_1 = require("@cubejs-backend/base-driver");
|
|
6
7
|
const cubestore_driver_1 = require("@cubejs-backend/cubestore-driver");
|
|
7
8
|
const TimeoutError_1 = require("./TimeoutError");
|
|
8
9
|
const ContinueWaitError_1 = require("./ContinueWaitError");
|
|
@@ -53,7 +54,7 @@ class QueryQueue {
|
|
|
53
54
|
this.executionTimeout = options.executionTimeout || (0, shared_1.getEnv)('dbQueryTimeout');
|
|
54
55
|
this.orphanedTimeout = options.orphanedTimeout || 120;
|
|
55
56
|
this.heartBeatInterval = options.heartBeatInterval || 30;
|
|
56
|
-
this.sendProcessMessageFn = options.sendProcessMessageFn || ((
|
|
57
|
+
this.sendProcessMessageFn = options.sendProcessMessageFn || ((queryKeyHash, queueId, retrieved) => { this.executeQuery(queryKeyHash, queueId, retrieved); });
|
|
57
58
|
this.sendCancelMessageFn = options.sendCancelMessageFn || ((query, queueId) => { this.processCancel(query, queueId); });
|
|
58
59
|
this.queryHandlers = options.queryHandlers;
|
|
59
60
|
this.streamHandler = options.streamHandler;
|
|
@@ -96,7 +97,8 @@ class QueryQueue {
|
|
|
96
97
|
this.streamEvents.emit('streamStarted', key);
|
|
97
98
|
return stream;
|
|
98
99
|
}
|
|
99
|
-
|
|
100
|
+
// Zero is falsy, and a queueId is still checked for truthiness in a few places
|
|
101
|
+
counter = 1;
|
|
100
102
|
generateQueueId() {
|
|
101
103
|
return this.counter++;
|
|
102
104
|
}
|
|
@@ -107,7 +109,7 @@ class QueryQueue {
|
|
|
107
109
|
*
|
|
108
110
|
* @throw {ContinueWaitError}
|
|
109
111
|
*/
|
|
110
|
-
async executeInQueue(queryHandler, queryKey, query, priority, executeOptions) {
|
|
112
|
+
async executeInQueue(queryHandler, queryKey, query, priority = base_driver_1.QueuePriority.Background, executeOptions) {
|
|
111
113
|
const options = {
|
|
112
114
|
queueId: this.generateQueueId(),
|
|
113
115
|
...executeOptions,
|
|
@@ -143,10 +145,8 @@ class QueryQueue {
|
|
|
143
145
|
}
|
|
144
146
|
const queueConnection = await this.queueDriver.createConnection();
|
|
145
147
|
let waitingContext;
|
|
148
|
+
let streamWait = null;
|
|
146
149
|
try {
|
|
147
|
-
if (priority == null) {
|
|
148
|
-
priority = 0;
|
|
149
|
-
}
|
|
150
150
|
if (!(priority >= -10000 && priority <= 10000)) {
|
|
151
151
|
throw new Error('Priority should be between -10000 and 10000');
|
|
152
152
|
}
|
|
@@ -163,12 +163,8 @@ class QueryQueue {
|
|
|
163
163
|
if (jobExists)
|
|
164
164
|
return null;
|
|
165
165
|
}
|
|
166
|
-
const time = new Date().getTime();
|
|
167
|
-
const keyScore = time + (10000 - priority) * 1E14;
|
|
168
166
|
options.orphanedTimeout = query.orphanedTimeout;
|
|
169
|
-
const
|
|
170
|
-
const orphanedTime = time + (orphanedTimeout * 1000);
|
|
171
|
-
const [added, queueId, queueSize, addedToQueueTime] = await queueConnection.addToQueue(keyScore, queryKey, orphanedTime, queryHandler, query, priority, options);
|
|
167
|
+
const [added, queueId, queueSize, addedToQueueTime, retrieved] = await queueConnection.addToQueue(queryKey, queryHandler, query, priority, options);
|
|
172
168
|
if (added > 0) {
|
|
173
169
|
waitingContext = {
|
|
174
170
|
queueId,
|
|
@@ -193,11 +189,23 @@ class QueryQueue {
|
|
|
193
189
|
preAggregation: query.preAggregation,
|
|
194
190
|
addedToQueueTime,
|
|
195
191
|
persistent: !!queryKey.persistent,
|
|
192
|
+
fastTrack: !!retrieved,
|
|
196
193
|
});
|
|
197
194
|
}
|
|
198
|
-
|
|
195
|
+
// Subscribing after the dispatch would lose the `streamStarted` event of a handler
|
|
196
|
+
// which starts fast
|
|
197
|
+
if (queryHandler === 'stream') {
|
|
198
|
+
streamWait = this.waitForQueryStream(queryKeyHash);
|
|
199
|
+
}
|
|
200
|
+
if (retrieved) {
|
|
201
|
+
// The item is active already, there is nothing for reconcile to pick up
|
|
202
|
+
await this.dispatchQuery(queryKeyHash, queueId, retrieved);
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
await this.reconcileQueue();
|
|
206
|
+
}
|
|
199
207
|
if (!added) {
|
|
200
|
-
const queryDef = await queueConnection.getQueryDef(queryKeyHash, queueId);
|
|
208
|
+
const queryDef = retrieved ? retrieved.def : await queueConnection.getQueryDef(queryKeyHash, queueId);
|
|
201
209
|
if (queryDef) {
|
|
202
210
|
waitingContext = {
|
|
203
211
|
queueId,
|
|
@@ -209,44 +217,20 @@ class QueryQueue {
|
|
|
209
217
|
};
|
|
210
218
|
}
|
|
211
219
|
}
|
|
212
|
-
|
|
220
|
+
// A retrieval carries the active keys of its prefix, and a retrieved query is never pending,
|
|
221
|
+
// so it has no place in the queue to report
|
|
222
|
+
const [active, toProcess] = retrieved ? [retrieved.active, undefined] : await queueConnection.getQueryStageState(true);
|
|
213
223
|
this.logger('Waiting for query', {
|
|
214
224
|
...waitingContext,
|
|
215
225
|
queueSize,
|
|
216
226
|
activeQueryKeys: active,
|
|
217
227
|
toProcessQueryKeys: toProcess,
|
|
218
228
|
active: active.indexOf(queryKeyHash) !== -1,
|
|
219
|
-
queueIndex: toProcess.indexOf(queryKeyHash),
|
|
229
|
+
queueIndex: toProcess ? toProcess.indexOf(queryKeyHash) : -1,
|
|
230
|
+
fastTrack: !!retrieved,
|
|
220
231
|
});
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
// TODO: Fix an issue with a fast execution of stream handler which caused by removal of QueryStream from streams,
|
|
224
|
-
// while EventListener doesnt start to listen for started stream event
|
|
225
|
-
if (queryHandler === 'stream') {
|
|
226
|
-
const self = this;
|
|
227
|
-
result = await new Promise((resolve) => {
|
|
228
|
-
let timeoutTimerId = null;
|
|
229
|
-
const onStreamStarted = (streamStartedHash) => {
|
|
230
|
-
if (streamStartedHash === queryKeyHash) {
|
|
231
|
-
if (timeoutTimerId) {
|
|
232
|
-
clearTimeout(timeoutTimerId);
|
|
233
|
-
}
|
|
234
|
-
resolve(self.getQueryStream(queryKeyHash));
|
|
235
|
-
}
|
|
236
|
-
};
|
|
237
|
-
self.streamEvents.addListener('streamStarted', onStreamStarted);
|
|
238
|
-
const stream = this.getQueryStream(queryKeyHash);
|
|
239
|
-
if (stream) {
|
|
240
|
-
self.streamEvents.removeListener('streamStarted', onStreamStarted);
|
|
241
|
-
resolve(stream);
|
|
242
|
-
}
|
|
243
|
-
else {
|
|
244
|
-
timeoutTimerId = setTimeout(() => {
|
|
245
|
-
self.streamEvents.removeListener('streamStarted', onStreamStarted);
|
|
246
|
-
resolve(null);
|
|
247
|
-
}, this.continueWaitTimeout * 10000);
|
|
248
|
-
}
|
|
249
|
-
});
|
|
232
|
+
if (streamWait) {
|
|
233
|
+
result = await streamWait.promise;
|
|
250
234
|
}
|
|
251
235
|
else {
|
|
252
236
|
// Result here won't be fetched for a jobed build query (initialized by
|
|
@@ -266,9 +250,48 @@ class QueryQueue {
|
|
|
266
250
|
throw error;
|
|
267
251
|
}
|
|
268
252
|
finally {
|
|
253
|
+
streamWait?.dispose();
|
|
269
254
|
this.queueDriver.release(queueConnection);
|
|
270
255
|
}
|
|
271
256
|
}
|
|
257
|
+
/**
|
|
258
|
+
* `dispose` releases the listener and the timer, it's a no-op once the promise resolved.
|
|
259
|
+
*/
|
|
260
|
+
waitForQueryStream(queryKeyHash) {
|
|
261
|
+
let timeoutTimerId = null;
|
|
262
|
+
let onStreamStarted = null;
|
|
263
|
+
const dispose = () => {
|
|
264
|
+
if (timeoutTimerId) {
|
|
265
|
+
clearTimeout(timeoutTimerId);
|
|
266
|
+
timeoutTimerId = null;
|
|
267
|
+
}
|
|
268
|
+
if (onStreamStarted) {
|
|
269
|
+
this.streamEvents.removeListener('streamStarted', onStreamStarted);
|
|
270
|
+
onStreamStarted = null;
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
const promise = new Promise((resolve) => {
|
|
274
|
+
onStreamStarted = (streamStartedHash) => {
|
|
275
|
+
if (streamStartedHash === queryKeyHash) {
|
|
276
|
+
dispose();
|
|
277
|
+
resolve(this.getQueryStream(queryKeyHash) ?? null);
|
|
278
|
+
}
|
|
279
|
+
};
|
|
280
|
+
this.streamEvents.addListener('streamStarted', onStreamStarted);
|
|
281
|
+
const stream = this.getQueryStream(queryKeyHash);
|
|
282
|
+
if (stream) {
|
|
283
|
+
dispose();
|
|
284
|
+
resolve(stream);
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
timeoutTimerId = setTimeout(() => {
|
|
288
|
+
dispose();
|
|
289
|
+
resolve(null);
|
|
290
|
+
}, this.continueWaitTimeout * 10000);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
return { promise, dispose };
|
|
294
|
+
}
|
|
272
295
|
/**
|
|
273
296
|
* @throw {Error}
|
|
274
297
|
*/
|
|
@@ -422,7 +445,7 @@ class QueryQueue {
|
|
|
422
445
|
const [queryDef] = await queueConnection.getQueryAndRemove(queryKey, queueId);
|
|
423
446
|
if (queryDef) {
|
|
424
447
|
this.logger('Removing orphaned query', {
|
|
425
|
-
queueId
|
|
448
|
+
queueId,
|
|
426
449
|
queryKey: queryDef.queryKey,
|
|
427
450
|
queuePrefix: this.redisQueuePrefix,
|
|
428
451
|
requestId: queryDef.requestId,
|
|
@@ -461,7 +484,8 @@ class QueryQueue {
|
|
|
461
484
|
}
|
|
462
485
|
})
|
|
463
486
|
.slice(0, toProcessLimit)
|
|
464
|
-
.map(([queryKey, queueId]) => this.
|
|
487
|
+
.map(([queryKey, queueId]) => this.processQuery(queryKey, queueId));
|
|
488
|
+
// Awaits the retrieval of every picked query, not their execution.
|
|
465
489
|
await Promise.all(tasks);
|
|
466
490
|
}
|
|
467
491
|
finally {
|
|
@@ -592,256 +616,263 @@ class QueryQueue {
|
|
|
592
616
|
return executionResult;
|
|
593
617
|
}
|
|
594
618
|
/**
|
|
595
|
-
*
|
|
596
|
-
* of the logic related to the queue updates, heartbeat, etc.
|
|
619
|
+
* Retrieves the query specified by the `queryKeyHashed` and hands it over for execution.
|
|
597
620
|
*/
|
|
598
621
|
async processQuery(queryKeyHashed, queueId) {
|
|
622
|
+
const retrieved = await this.retrieveQueryForProcessing(queryKeyHashed, queueId);
|
|
623
|
+
if (!retrieved) {
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
await this.dispatchQuery(queryKeyHashed, queueId, retrieved);
|
|
627
|
+
}
|
|
628
|
+
async dispatchQuery(queryKeyHash, queueId, retrieved) {
|
|
629
|
+
try {
|
|
630
|
+
await this.sendProcessMessageFn(queryKeyHash, queueId, retrieved);
|
|
631
|
+
}
|
|
632
|
+
catch (e) {
|
|
633
|
+
this.logger('Error while processing message', {
|
|
634
|
+
queueId,
|
|
635
|
+
queryKey: retrieved.def.queryKey,
|
|
636
|
+
requestId: retrieved.def.requestId,
|
|
637
|
+
error: (e.stack || e).toString(),
|
|
638
|
+
queuePrefix: this.redisQueuePrefix
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
/**
|
|
643
|
+
* Atomically moves the query specified by `queryKeyHashed` to the active set. Returns `null`
|
|
644
|
+
* when another node is already running the query or the concurrency budget is full.
|
|
645
|
+
*/
|
|
646
|
+
async retrieveQueryForProcessing(queryKeyHashed, queueId) {
|
|
599
647
|
const queueConnection = await this.queueDriver.createConnection();
|
|
600
|
-
let insertedCount;
|
|
601
|
-
let activeKeys;
|
|
602
|
-
let queueSize;
|
|
603
|
-
let query;
|
|
604
|
-
let processingLockAcquired;
|
|
605
648
|
try {
|
|
606
|
-
const
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
let retrieveQueueId;
|
|
610
|
-
[insertedCount, retrieveQueueId, activeKeys, queueSize, query, processingLockAcquired] = retrieveResult;
|
|
611
|
-
// Backward compatibility for old Cube Store, Redis and Memory
|
|
612
|
-
if (retrieveQueueId) {
|
|
613
|
-
queueId = retrieveQueueId;
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
const activated = activeKeys && activeKeys.indexOf(queryKeyHashed) !== -1;
|
|
617
|
-
if (!query) {
|
|
618
|
-
query = await queueConnection.getQueryDef(queryKeyHashed, null);
|
|
619
|
-
}
|
|
620
|
-
if (query && insertedCount && activated && processingLockAcquired) {
|
|
621
|
-
let executionResult;
|
|
622
|
-
let queryExecutionFinished = false;
|
|
623
|
-
// Set by the query handler's setCancelHandler callback once execution begins.
|
|
624
|
-
// Not available on the original query def from retrieveForProcessing.
|
|
625
|
-
let localCancelHandler = null;
|
|
626
|
-
const startQueryTime = (new Date()).getTime();
|
|
627
|
-
const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
|
|
628
|
-
this.logger('Performing query', {
|
|
649
|
+
const retrieved = await queueConnection.retrieveForProcessing(queryKeyHashed, queueId);
|
|
650
|
+
if (!retrieved) {
|
|
651
|
+
this.logger('Skip processing', {
|
|
629
652
|
queueId,
|
|
630
|
-
|
|
631
|
-
queueSize,
|
|
632
|
-
queryKey: query.queryKey,
|
|
653
|
+
queryKey: queryKeyHashed,
|
|
633
654
|
queuePrefix: this.redisQueuePrefix,
|
|
634
|
-
requestId: query.requestId,
|
|
635
|
-
timeInQueue,
|
|
636
|
-
metadata: query.query?.metadata,
|
|
637
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
638
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
639
|
-
preAggregation: query.query?.preAggregation,
|
|
640
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
641
655
|
});
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
}
|
|
694
|
-
}, this.heartBeatInterval * 1000);
|
|
695
|
-
try {
|
|
696
|
-
const handler = query?.queryHandler;
|
|
697
|
-
switch (handler) {
|
|
698
|
-
case 'stream':
|
|
699
|
-
// eslint-disable-next-line no-case-declarations
|
|
700
|
-
const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
|
|
701
|
-
try {
|
|
702
|
-
await this.streamHandler(query.query, queryStream);
|
|
703
|
-
// CubeStore has special handling for null
|
|
704
|
-
executionResult = {
|
|
705
|
-
streamResult: true
|
|
706
|
-
};
|
|
707
|
-
}
|
|
708
|
-
finally {
|
|
709
|
-
if (this.streams.get(queryKeyHashed) === queryStream) {
|
|
710
|
-
this.streams.delete(queryKeyHashed);
|
|
711
|
-
}
|
|
712
|
-
}
|
|
713
|
-
break;
|
|
714
|
-
default:
|
|
715
|
-
executionResult = {
|
|
716
|
-
result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
|
|
717
|
-
localCancelHandler = cancelHandler;
|
|
718
|
-
try {
|
|
719
|
-
await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, processingId, queueId);
|
|
720
|
-
}
|
|
721
|
-
catch (e) {
|
|
722
|
-
this.logger('Error while query update', {
|
|
723
|
-
queueId,
|
|
724
|
-
queryKey: query.queryKey,
|
|
725
|
-
error: e.stack || e,
|
|
726
|
-
queuePrefix: this.redisQueuePrefix,
|
|
727
|
-
requestId: query.requestId,
|
|
728
|
-
metadata: query.query?.metadata,
|
|
729
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
730
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
731
|
-
preAggregation: query.query?.preAggregation,
|
|
732
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
733
|
-
});
|
|
734
|
-
}
|
|
735
|
-
}))
|
|
736
|
-
};
|
|
737
|
-
break;
|
|
738
|
-
}
|
|
739
|
-
this.logger('Performing query completed', {
|
|
656
|
+
return null;
|
|
657
|
+
}
|
|
658
|
+
return retrieved;
|
|
659
|
+
}
|
|
660
|
+
catch (e) {
|
|
661
|
+
this.logger('Queue storage error', {
|
|
662
|
+
queueId,
|
|
663
|
+
queryKey: queryKeyHashed,
|
|
664
|
+
error: (e.stack || e).toString(),
|
|
665
|
+
queuePrefix: this.redisQueuePrefix
|
|
666
|
+
});
|
|
667
|
+
return null;
|
|
668
|
+
}
|
|
669
|
+
finally {
|
|
670
|
+
this.queueDriver.release(queueConnection);
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
/**
|
|
674
|
+
* Executes a retrieved query: runs its handler while keeping the queue heartbeat alive, then acks
|
|
675
|
+
* the result. It's the counterpart of `sendProcessMessageFn` and the entry point for a custom
|
|
676
|
+
* implementation which hands the query over to another process.
|
|
677
|
+
*/
|
|
678
|
+
async executeQuery(queryKeyHashed, queueId, retrieved) {
|
|
679
|
+
const { queueSize, def: query } = retrieved;
|
|
680
|
+
const queueConnection = await this.queueDriver.createConnection();
|
|
681
|
+
try {
|
|
682
|
+
let executionResult;
|
|
683
|
+
let queryExecutionFinished = false;
|
|
684
|
+
// Set by the query handler's setCancelHandler callback once execution begins.
|
|
685
|
+
// Not available on the original query def from retrieveForProcessing.
|
|
686
|
+
let localCancelHandler = null;
|
|
687
|
+
const startQueryTime = (new Date()).getTime();
|
|
688
|
+
const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
|
|
689
|
+
this.logger('Performing query', {
|
|
690
|
+
queueId,
|
|
691
|
+
queueSize,
|
|
692
|
+
queryKey: query.queryKey,
|
|
693
|
+
queuePrefix: this.redisQueuePrefix,
|
|
694
|
+
requestId: query.requestId,
|
|
695
|
+
timeInQueue,
|
|
696
|
+
metadata: query.query?.metadata,
|
|
697
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
698
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
699
|
+
preAggregation: query.query?.preAggregation,
|
|
700
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
701
|
+
});
|
|
702
|
+
await queueConnection.optimisticQueryUpdate(queryKeyHashed, { startQueryTime }, queueId);
|
|
703
|
+
let queryProcessHeartbeat = Date.now();
|
|
704
|
+
const heartBeatTimer = setInterval(async () => {
|
|
705
|
+
if ((Date.now() - queryProcessHeartbeat) > 5 * 60 * 1000) {
|
|
706
|
+
this.logger('Query processing heartbeat', {
|
|
740
707
|
queueId,
|
|
741
|
-
processingId,
|
|
742
|
-
queueSize,
|
|
743
|
-
duration: ((new Date()).getTime() - startQueryTime),
|
|
744
708
|
queryKey: query.queryKey,
|
|
745
|
-
queuePrefix: this.redisQueuePrefix,
|
|
746
709
|
requestId: query.requestId,
|
|
747
|
-
timeInQueue,
|
|
748
|
-
metadata: query.query?.metadata,
|
|
749
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
750
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
751
|
-
preAggregation: query.query?.preAggregation,
|
|
752
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
753
710
|
});
|
|
711
|
+
queryProcessHeartbeat = Date.now();
|
|
712
|
+
}
|
|
713
|
+
try {
|
|
714
|
+
await queueConnection.updateHeartBeat(queryKeyHashed, queueId);
|
|
754
715
|
}
|
|
755
716
|
catch (e) {
|
|
756
|
-
|
|
757
|
-
error: (e.message || e).toString() // TODO error handling
|
|
758
|
-
};
|
|
759
|
-
this.logger('Error while querying', {
|
|
717
|
+
this.logger('Error updating heartbeat', {
|
|
760
718
|
queueId,
|
|
761
|
-
processingId,
|
|
762
|
-
queueSize,
|
|
763
|
-
duration: ((new Date()).getTime() - startQueryTime),
|
|
764
719
|
queryKey: query.queryKey,
|
|
720
|
+
error: e.stack || e,
|
|
765
721
|
queuePrefix: this.redisQueuePrefix,
|
|
766
722
|
requestId: query.requestId,
|
|
767
|
-
timeInQueue,
|
|
768
|
-
metadata: query.query?.metadata,
|
|
769
|
-
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
770
|
-
newVersionEntry: query.query?.newVersionEntry,
|
|
771
|
-
preAggregation: query.query?.preAggregation,
|
|
772
|
-
addedToQueueTime: query.addedToQueueTime,
|
|
773
|
-
error: (e.stack || e).toString()
|
|
774
723
|
});
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
724
|
+
}
|
|
725
|
+
if (!queryExecutionFinished && localCancelHandler !== null) {
|
|
726
|
+
try {
|
|
727
|
+
const currentDef = await queueConnection.getQueryDef(queryKeyHashed, queueId);
|
|
728
|
+
if (!currentDef && !queryExecutionFinished) {
|
|
729
|
+
this.logger('Cancelling query due to external cancellation', {
|
|
779
730
|
queueId,
|
|
780
|
-
|
|
781
|
-
queryKey: queryWithCancelHandle.queryKey,
|
|
731
|
+
queryKey: query.queryKey,
|
|
782
732
|
queuePrefix: this.redisQueuePrefix,
|
|
783
|
-
requestId:
|
|
784
|
-
metadata:
|
|
785
|
-
preAggregationId:
|
|
786
|
-
newVersionEntry:
|
|
787
|
-
preAggregation:
|
|
788
|
-
addedToQueueTime:
|
|
733
|
+
requestId: query.requestId,
|
|
734
|
+
metadata: query.query?.metadata,
|
|
735
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
736
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
737
|
+
preAggregation: query.query?.preAggregation,
|
|
738
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
789
739
|
});
|
|
790
|
-
|
|
740
|
+
const cancelQuery = { ...query, cancelHandler: localCancelHandler };
|
|
741
|
+
await this.processCancel(cancelQuery, queueId);
|
|
791
742
|
}
|
|
792
743
|
}
|
|
744
|
+
catch (e) {
|
|
745
|
+
this.logger('Error checking for external cancellation', {
|
|
746
|
+
queueId,
|
|
747
|
+
queryKey: query.queryKey,
|
|
748
|
+
error: e.stack || e,
|
|
749
|
+
queuePrefix: this.redisQueuePrefix,
|
|
750
|
+
requestId: query.requestId,
|
|
751
|
+
});
|
|
752
|
+
}
|
|
793
753
|
}
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
754
|
+
}, this.heartBeatInterval * 1000);
|
|
755
|
+
try {
|
|
756
|
+
const handler = query?.queryHandler;
|
|
757
|
+
switch (handler) {
|
|
758
|
+
case 'stream':
|
|
759
|
+
// eslint-disable-next-line no-case-declarations
|
|
760
|
+
const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
|
|
761
|
+
try {
|
|
762
|
+
await this.streamHandler(query.query, queryStream);
|
|
763
|
+
// CubeStore has special handling for null
|
|
764
|
+
executionResult = {
|
|
765
|
+
streamResult: true
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
finally {
|
|
769
|
+
if (this.streams.get(queryKeyHashed) === queryStream) {
|
|
770
|
+
this.streams.delete(queryKeyHashed);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
break;
|
|
774
|
+
default:
|
|
775
|
+
executionResult = {
|
|
776
|
+
result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
|
|
777
|
+
localCancelHandler = cancelHandler;
|
|
778
|
+
try {
|
|
779
|
+
await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, queueId);
|
|
780
|
+
}
|
|
781
|
+
catch (e) {
|
|
782
|
+
this.logger('Error while query update', {
|
|
783
|
+
queueId,
|
|
784
|
+
queryKey: query.queryKey,
|
|
785
|
+
error: e.stack || e,
|
|
786
|
+
queuePrefix: this.redisQueuePrefix,
|
|
787
|
+
requestId: query.requestId,
|
|
788
|
+
metadata: query.query?.metadata,
|
|
789
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
790
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
791
|
+
preAggregation: query.query?.preAggregation,
|
|
792
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
}))
|
|
796
|
+
};
|
|
797
|
+
break;
|
|
797
798
|
}
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
799
|
+
this.logger('Performing query completed', {
|
|
800
|
+
queueId,
|
|
801
|
+
queueSize,
|
|
802
|
+
duration: ((new Date()).getTime() - startQueryTime),
|
|
803
|
+
queryKey: query.queryKey,
|
|
804
|
+
queuePrefix: this.redisQueuePrefix,
|
|
805
|
+
requestId: query.requestId,
|
|
806
|
+
timeInQueue,
|
|
807
|
+
metadata: query.query?.metadata,
|
|
808
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
809
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
810
|
+
preAggregation: query.query?.preAggregation,
|
|
811
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
catch (e) {
|
|
815
|
+
executionResult = {
|
|
816
|
+
error: (e.message || e).toString() // TODO error handling
|
|
817
|
+
};
|
|
818
|
+
this.logger('Error while querying', {
|
|
819
|
+
queueId,
|
|
820
|
+
queueSize,
|
|
821
|
+
duration: ((new Date()).getTime() - startQueryTime),
|
|
822
|
+
queryKey: query.queryKey,
|
|
823
|
+
queuePrefix: this.redisQueuePrefix,
|
|
824
|
+
requestId: query.requestId,
|
|
825
|
+
timeInQueue,
|
|
826
|
+
metadata: query.query?.metadata,
|
|
827
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
828
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
829
|
+
preAggregation: query.query?.preAggregation,
|
|
830
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
831
|
+
error: (e.stack || e).toString()
|
|
832
|
+
});
|
|
833
|
+
if (e instanceof TimeoutError_1.TimeoutError) {
|
|
834
|
+
const queryWithCancelHandle = await queueConnection.getQueryDef(queryKeyHashed, queueId);
|
|
835
|
+
if (queryWithCancelHandle) {
|
|
836
|
+
this.logger('Cancelling query due to timeout', {
|
|
837
|
+
queueId,
|
|
838
|
+
queryKey: queryWithCancelHandle.queryKey,
|
|
839
|
+
queuePrefix: this.redisQueuePrefix,
|
|
840
|
+
requestId: queryWithCancelHandle.requestId,
|
|
841
|
+
metadata: queryWithCancelHandle.query?.metadata,
|
|
842
|
+
preAggregationId: queryWithCancelHandle.query?.preAggregation?.preAggregationId,
|
|
843
|
+
newVersionEntry: queryWithCancelHandle.query?.newVersionEntry,
|
|
844
|
+
preAggregation: queryWithCancelHandle.query?.preAggregation,
|
|
845
|
+
addedToQueueTime: queryWithCancelHandle.addedToQueueTime,
|
|
846
|
+
});
|
|
847
|
+
await this.sendCancelMessageFn(queryWithCancelHandle, queueId);
|
|
848
|
+
}
|
|
812
849
|
}
|
|
813
|
-
await this.reconcileQueue();
|
|
814
850
|
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
// await this.reconcileQueue();
|
|
822
|
-
// }
|
|
823
|
-
// }
|
|
824
|
-
this.logger('Skip processing', {
|
|
851
|
+
finally {
|
|
852
|
+
queryExecutionFinished = true;
|
|
853
|
+
clearInterval(heartBeatTimer);
|
|
854
|
+
}
|
|
855
|
+
if (!(await queueConnection.setResultAndRemoveQuery(queryKeyHashed, executionResult, queueId))) {
|
|
856
|
+
this.logger('Orphaned execution result', {
|
|
825
857
|
queueId,
|
|
826
|
-
|
|
827
|
-
queryKey: query
|
|
828
|
-
requestId: query && query.requestId,
|
|
858
|
+
warn: 'Result for query was not set because the queue item is no longer active',
|
|
859
|
+
queryKey: query.queryKey,
|
|
829
860
|
queuePrefix: this.redisQueuePrefix,
|
|
830
|
-
|
|
831
|
-
query,
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
861
|
+
requestId: query.requestId,
|
|
862
|
+
metadata: query.query?.metadata,
|
|
863
|
+
preAggregationId: query.query?.preAggregation?.preAggregationId,
|
|
864
|
+
newVersionEntry: query.query?.newVersionEntry,
|
|
865
|
+
preAggregation: query.query?.preAggregation,
|
|
866
|
+
addedToQueueTime: query.addedToQueueTime,
|
|
836
867
|
});
|
|
837
|
-
await queueConnection.freeProcessingLock(queryKeyHashed, processingId, activated);
|
|
838
868
|
}
|
|
869
|
+
await this.reconcileQueue();
|
|
839
870
|
}
|
|
840
871
|
catch (e) {
|
|
841
872
|
this.logger('Queue storage error', {
|
|
842
873
|
queueId,
|
|
843
|
-
queryKey: query
|
|
844
|
-
requestId: query
|
|
874
|
+
queryKey: query.queryKey,
|
|
875
|
+
requestId: query.requestId,
|
|
845
876
|
error: (e.stack || e).toString(),
|
|
846
877
|
queuePrefix: this.redisQueuePrefix
|
|
847
878
|
});
|