@cubejs-backend/query-orchestrator 1.7.25 → 1.7.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.QueryQueue = void 0;
4
4
  const events_1 = require("events");
5
5
  const shared_1 = require("@cubejs-backend/shared");
6
+ const base_driver_1 = require("@cubejs-backend/base-driver");
6
7
  const cubestore_driver_1 = require("@cubejs-backend/cubestore-driver");
7
8
  const TimeoutError_1 = require("./TimeoutError");
8
9
  const ContinueWaitError_1 = require("./ContinueWaitError");
@@ -53,7 +54,7 @@ class QueryQueue {
53
54
  this.executionTimeout = options.executionTimeout || (0, shared_1.getEnv)('dbQueryTimeout');
54
55
  this.orphanedTimeout = options.orphanedTimeout || 120;
55
56
  this.heartBeatInterval = options.heartBeatInterval || 30;
56
- this.sendProcessMessageFn = options.sendProcessMessageFn || ((queryKey, queryId) => { this.processQuery(queryKey, queryId); });
57
+ this.sendProcessMessageFn = options.sendProcessMessageFn || ((queryKeyHash, queueId, retrieved) => { this.executeQuery(queryKeyHash, queueId, retrieved); });
57
58
  this.sendCancelMessageFn = options.sendCancelMessageFn || ((query, queueId) => { this.processCancel(query, queueId); });
58
59
  this.queryHandlers = options.queryHandlers;
59
60
  this.streamHandler = options.streamHandler;
@@ -96,7 +97,8 @@ class QueryQueue {
96
97
  this.streamEvents.emit('streamStarted', key);
97
98
  return stream;
98
99
  }
99
- counter = 0;
100
+ // Zero is falsy, and a queueId is still checked for truthiness in a few places
101
+ counter = 1;
100
102
  generateQueueId() {
101
103
  return this.counter++;
102
104
  }
@@ -107,7 +109,7 @@ class QueryQueue {
107
109
  *
108
110
  * @throw {ContinueWaitError}
109
111
  */
110
- async executeInQueue(queryHandler, queryKey, query, priority, executeOptions) {
112
+ async executeInQueue(queryHandler, queryKey, query, priority = base_driver_1.QueuePriority.Background, executeOptions) {
111
113
  const options = {
112
114
  queueId: this.generateQueueId(),
113
115
  ...executeOptions,
@@ -143,10 +145,8 @@ class QueryQueue {
143
145
  }
144
146
  const queueConnection = await this.queueDriver.createConnection();
145
147
  let waitingContext;
148
+ let streamWait = null;
146
149
  try {
147
- if (priority == null) {
148
- priority = 0;
149
- }
150
150
  if (!(priority >= -10000 && priority <= 10000)) {
151
151
  throw new Error('Priority should be between -10000 and 10000');
152
152
  }
@@ -163,12 +163,8 @@ class QueryQueue {
163
163
  if (jobExists)
164
164
  return null;
165
165
  }
166
- const time = new Date().getTime();
167
- const keyScore = time + (10000 - priority) * 1E14;
168
166
  options.orphanedTimeout = query.orphanedTimeout;
169
- const orphanedTimeout = 'orphanedTimeout' in query ? query.orphanedTimeout : this.orphanedTimeout;
170
- const orphanedTime = time + (orphanedTimeout * 1000);
171
- const [added, queueId, queueSize, addedToQueueTime] = await queueConnection.addToQueue(keyScore, queryKey, orphanedTime, queryHandler, query, priority, options);
167
+ const [added, queueId, queueSize, addedToQueueTime, retrieved] = await queueConnection.addToQueue(queryKey, queryHandler, query, priority, options);
172
168
  if (added > 0) {
173
169
  waitingContext = {
174
170
  queueId,
@@ -193,11 +189,23 @@ class QueryQueue {
193
189
  preAggregation: query.preAggregation,
194
190
  addedToQueueTime,
195
191
  persistent: !!queryKey.persistent,
192
+ fastTrack: !!retrieved,
196
193
  });
197
194
  }
198
- await this.reconcileQueue();
195
+ // Subscribing after the dispatch would lose the `streamStarted` event of a handler
196
+ // which starts fast
197
+ if (queryHandler === 'stream') {
198
+ streamWait = this.waitForQueryStream(queryKeyHash);
199
+ }
200
+ if (retrieved) {
201
+ // The item is active already, there is nothing for reconcile to pick up
202
+ await this.dispatchQuery(queryKeyHash, queueId, retrieved);
203
+ }
204
+ else {
205
+ await this.reconcileQueue();
206
+ }
199
207
  if (!added) {
200
- const queryDef = await queueConnection.getQueryDef(queryKeyHash, queueId);
208
+ const queryDef = retrieved ? retrieved.def : await queueConnection.getQueryDef(queryKeyHash, queueId);
201
209
  if (queryDef) {
202
210
  waitingContext = {
203
211
  queueId,
@@ -209,44 +217,20 @@ class QueryQueue {
209
217
  };
210
218
  }
211
219
  }
212
- const [active, toProcess] = await queueConnection.getQueryStageState(true);
220
+ // A retrieval carries the active keys of its prefix, and a retrieved query is never pending,
221
+ // so it has no place in the queue to report
222
+ const [active, toProcess] = retrieved ? [retrieved.active, undefined] : await queueConnection.getQueryStageState(true);
213
223
  this.logger('Waiting for query', {
214
224
  ...waitingContext,
215
225
  queueSize,
216
226
  activeQueryKeys: active,
217
227
  toProcessQueryKeys: toProcess,
218
228
  active: active.indexOf(queryKeyHash) !== -1,
219
- queueIndex: toProcess.indexOf(queryKeyHash),
229
+ queueIndex: toProcess ? toProcess.indexOf(queryKeyHash) : -1,
230
+ fastTrack: !!retrieved,
220
231
  });
221
- // Stream processing goes here under assumption there's no way of a stream close just after it was added to the `streams` map.
222
- // Otherwise `streamStarted` event listener should go before the `reconcileQueue` call.
223
- // TODO: Fix an issue with a fast execution of stream handler which caused by removal of QueryStream from streams,
224
- // while EventListener doesnt start to listen for started stream event
225
- if (queryHandler === 'stream') {
226
- const self = this;
227
- result = await new Promise((resolve) => {
228
- let timeoutTimerId = null;
229
- const onStreamStarted = (streamStartedHash) => {
230
- if (streamStartedHash === queryKeyHash) {
231
- if (timeoutTimerId) {
232
- clearTimeout(timeoutTimerId);
233
- }
234
- resolve(self.getQueryStream(queryKeyHash));
235
- }
236
- };
237
- self.streamEvents.addListener('streamStarted', onStreamStarted);
238
- const stream = this.getQueryStream(queryKeyHash);
239
- if (stream) {
240
- self.streamEvents.removeListener('streamStarted', onStreamStarted);
241
- resolve(stream);
242
- }
243
- else {
244
- timeoutTimerId = setTimeout(() => {
245
- self.streamEvents.removeListener('streamStarted', onStreamStarted);
246
- resolve(null);
247
- }, this.continueWaitTimeout * 10000);
248
- }
249
- });
232
+ if (streamWait) {
233
+ result = await streamWait.promise;
250
234
  }
251
235
  else {
252
236
  // Result here won't be fetched for a jobed build query (initialized by
@@ -266,9 +250,48 @@ class QueryQueue {
266
250
  throw error;
267
251
  }
268
252
  finally {
253
+ streamWait?.dispose();
269
254
  this.queueDriver.release(queueConnection);
270
255
  }
271
256
  }
257
+ /**
258
+ * `dispose` releases the listener and the timer, it's a no-op once the promise resolved.
259
+ */
260
+ waitForQueryStream(queryKeyHash) {
261
+ let timeoutTimerId = null;
262
+ let onStreamStarted = null;
263
+ const dispose = () => {
264
+ if (timeoutTimerId) {
265
+ clearTimeout(timeoutTimerId);
266
+ timeoutTimerId = null;
267
+ }
268
+ if (onStreamStarted) {
269
+ this.streamEvents.removeListener('streamStarted', onStreamStarted);
270
+ onStreamStarted = null;
271
+ }
272
+ };
273
+ const promise = new Promise((resolve) => {
274
+ onStreamStarted = (streamStartedHash) => {
275
+ if (streamStartedHash === queryKeyHash) {
276
+ dispose();
277
+ resolve(this.getQueryStream(queryKeyHash) ?? null);
278
+ }
279
+ };
280
+ this.streamEvents.addListener('streamStarted', onStreamStarted);
281
+ const stream = this.getQueryStream(queryKeyHash);
282
+ if (stream) {
283
+ dispose();
284
+ resolve(stream);
285
+ }
286
+ else {
287
+ timeoutTimerId = setTimeout(() => {
288
+ dispose();
289
+ resolve(null);
290
+ }, this.continueWaitTimeout * 10000);
291
+ }
292
+ });
293
+ return { promise, dispose };
294
+ }
272
295
  /**
273
296
  * @throw {Error}
274
297
  */
@@ -422,7 +445,7 @@ class QueryQueue {
422
445
  const [queryDef] = await queueConnection.getQueryAndRemove(queryKey, queueId);
423
446
  if (queryDef) {
424
447
  this.logger('Removing orphaned query', {
425
- queueId: queueId || queryDef.queueId /** Special handling for Redis */,
448
+ queueId,
426
449
  queryKey: queryDef.queryKey,
427
450
  queuePrefix: this.redisQueuePrefix,
428
451
  requestId: queryDef.requestId,
@@ -461,7 +484,8 @@ class QueryQueue {
461
484
  }
462
485
  })
463
486
  .slice(0, toProcessLimit)
464
- .map(([queryKey, queueId]) => this.sendProcessMessageFn(queryKey, queueId));
487
+ .map(([queryKey, queueId]) => this.processQuery(queryKey, queueId));
488
+ // Awaits the retrieval of every picked query, not their execution.
465
489
  await Promise.all(tasks);
466
490
  }
467
491
  finally {
@@ -592,256 +616,263 @@ class QueryQueue {
592
616
  return executionResult;
593
617
  }
594
618
  /**
595
- * Processing query specified by the `queryKey`. This method encapsulates most
596
- * of the logic related to the queue updates, heartbeat, etc.
619
+ * Retrieves the query specified by the `queryKeyHashed` and hands it over for execution.
597
620
  */
598
621
  async processQuery(queryKeyHashed, queueId) {
622
+ const retrieved = await this.retrieveQueryForProcessing(queryKeyHashed, queueId);
623
+ if (!retrieved) {
624
+ return;
625
+ }
626
+ await this.dispatchQuery(queryKeyHashed, queueId, retrieved);
627
+ }
628
+ async dispatchQuery(queryKeyHash, queueId, retrieved) {
629
+ try {
630
+ await this.sendProcessMessageFn(queryKeyHash, queueId, retrieved);
631
+ }
632
+ catch (e) {
633
+ this.logger('Error while processing message', {
634
+ queueId,
635
+ queryKey: retrieved.def.queryKey,
636
+ requestId: retrieved.def.requestId,
637
+ error: (e.stack || e).toString(),
638
+ queuePrefix: this.redisQueuePrefix
639
+ });
640
+ }
641
+ }
642
+ /**
643
+ * Atomically moves the query specified by `queryKeyHashed` to the active set. Returns `null`
644
+ * when another node is already running the query or the concurrency budget is full.
645
+ */
646
+ async retrieveQueryForProcessing(queryKeyHashed, queueId) {
599
647
  const queueConnection = await this.queueDriver.createConnection();
600
- let insertedCount;
601
- let activeKeys;
602
- let queueSize;
603
- let query;
604
- let processingLockAcquired;
605
648
  try {
606
- const processingId = queueId || /** for Redis only */ await queueConnection.getNextProcessingId();
607
- const retrieveResult = await queueConnection.retrieveForProcessing(queryKeyHashed, processingId);
608
- if (retrieveResult) {
609
- let retrieveQueueId;
610
- [insertedCount, retrieveQueueId, activeKeys, queueSize, query, processingLockAcquired] = retrieveResult;
611
- // Backward compatibility for old Cube Store, Redis and Memory
612
- if (retrieveQueueId) {
613
- queueId = retrieveQueueId;
614
- }
615
- }
616
- const activated = activeKeys && activeKeys.indexOf(queryKeyHashed) !== -1;
617
- if (!query) {
618
- query = await queueConnection.getQueryDef(queryKeyHashed, null);
619
- }
620
- if (query && insertedCount && activated && processingLockAcquired) {
621
- let executionResult;
622
- let queryExecutionFinished = false;
623
- // Set by the query handler's setCancelHandler callback once execution begins.
624
- // Not available on the original query def from retrieveForProcessing.
625
- let localCancelHandler = null;
626
- const startQueryTime = (new Date()).getTime();
627
- const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
628
- this.logger('Performing query', {
649
+ const retrieved = await queueConnection.retrieveForProcessing(queryKeyHashed, queueId);
650
+ if (!retrieved) {
651
+ this.logger('Skip processing', {
629
652
  queueId,
630
- processingId,
631
- queueSize,
632
- queryKey: query.queryKey,
653
+ queryKey: queryKeyHashed,
633
654
  queuePrefix: this.redisQueuePrefix,
634
- requestId: query.requestId,
635
- timeInQueue,
636
- metadata: query.query?.metadata,
637
- preAggregationId: query.query?.preAggregation?.preAggregationId,
638
- newVersionEntry: query.query?.newVersionEntry,
639
- preAggregation: query.query?.preAggregation,
640
- addedToQueueTime: query.addedToQueueTime,
641
655
  });
642
- await queueConnection.optimisticQueryUpdate(queryKeyHashed, { startQueryTime }, processingId, queueId);
643
- let queryProcessHeartbeat = Date.now();
644
- const heartBeatTimer = setInterval(async () => {
645
- if ((Date.now() - queryProcessHeartbeat) > 5 * 60 * 1000) {
646
- this.logger('Query processing heartbeat', {
647
- queueId,
648
- queryKey: query.queryKey,
649
- requestId: query.requestId,
650
- });
651
- queryProcessHeartbeat = Date.now();
652
- }
653
- try {
654
- await queueConnection.updateHeartBeat(queryKeyHashed, queueId);
655
- }
656
- catch (e) {
657
- this.logger('Error updating heartbeat', {
658
- queueId,
659
- queryKey: query.queryKey,
660
- error: e.stack || e,
661
- queuePrefix: this.redisQueuePrefix,
662
- requestId: query.requestId,
663
- });
664
- }
665
- if (!queryExecutionFinished && localCancelHandler !== null) {
666
- try {
667
- const currentDef = await queueConnection.getQueryDef(queryKeyHashed, queueId);
668
- if (!currentDef && !queryExecutionFinished) {
669
- this.logger('Cancelling query due to external cancellation', {
670
- queueId,
671
- queryKey: query.queryKey,
672
- queuePrefix: this.redisQueuePrefix,
673
- requestId: query.requestId,
674
- metadata: query.query?.metadata,
675
- preAggregationId: query.query?.preAggregation?.preAggregationId,
676
- newVersionEntry: query.query?.newVersionEntry,
677
- preAggregation: query.query?.preAggregation,
678
- addedToQueueTime: query.addedToQueueTime,
679
- });
680
- const cancelQuery = { ...query, cancelHandler: localCancelHandler };
681
- await this.processCancel(cancelQuery, queueId);
682
- }
683
- }
684
- catch (e) {
685
- this.logger('Error checking for external cancellation', {
686
- queueId,
687
- queryKey: query.queryKey,
688
- error: e.stack || e,
689
- queuePrefix: this.redisQueuePrefix,
690
- requestId: query.requestId,
691
- });
692
- }
693
- }
694
- }, this.heartBeatInterval * 1000);
695
- try {
696
- const handler = query?.queryHandler;
697
- switch (handler) {
698
- case 'stream':
699
- // eslint-disable-next-line no-case-declarations
700
- const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
701
- try {
702
- await this.streamHandler(query.query, queryStream);
703
- // CubeStore has special handling for null
704
- executionResult = {
705
- streamResult: true
706
- };
707
- }
708
- finally {
709
- if (this.streams.get(queryKeyHashed) === queryStream) {
710
- this.streams.delete(queryKeyHashed);
711
- }
712
- }
713
- break;
714
- default:
715
- executionResult = {
716
- result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
717
- localCancelHandler = cancelHandler;
718
- try {
719
- await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, processingId, queueId);
720
- }
721
- catch (e) {
722
- this.logger('Error while query update', {
723
- queueId,
724
- queryKey: query.queryKey,
725
- error: e.stack || e,
726
- queuePrefix: this.redisQueuePrefix,
727
- requestId: query.requestId,
728
- metadata: query.query?.metadata,
729
- preAggregationId: query.query?.preAggregation?.preAggregationId,
730
- newVersionEntry: query.query?.newVersionEntry,
731
- preAggregation: query.query?.preAggregation,
732
- addedToQueueTime: query.addedToQueueTime,
733
- });
734
- }
735
- }))
736
- };
737
- break;
738
- }
739
- this.logger('Performing query completed', {
656
+ return null;
657
+ }
658
+ return retrieved;
659
+ }
660
+ catch (e) {
661
+ this.logger('Queue storage error', {
662
+ queueId,
663
+ queryKey: queryKeyHashed,
664
+ error: (e.stack || e).toString(),
665
+ queuePrefix: this.redisQueuePrefix
666
+ });
667
+ return null;
668
+ }
669
+ finally {
670
+ this.queueDriver.release(queueConnection);
671
+ }
672
+ }
673
+ /**
674
+ * Executes a retrieved query: runs its handler while keeping the queue heartbeat alive, then acks
675
+ * the result. It's the counterpart of `sendProcessMessageFn` and the entry point for a custom
676
+ * implementation which hands the query over to another process.
677
+ */
678
+ async executeQuery(queryKeyHashed, queueId, retrieved) {
679
+ const { queueSize, def: query } = retrieved;
680
+ const queueConnection = await this.queueDriver.createConnection();
681
+ try {
682
+ let executionResult;
683
+ let queryExecutionFinished = false;
684
+ // Set by the query handler's setCancelHandler callback once execution begins.
685
+ // Not available on the original query def from retrieveForProcessing.
686
+ let localCancelHandler = null;
687
+ const startQueryTime = (new Date()).getTime();
688
+ const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
689
+ this.logger('Performing query', {
690
+ queueId,
691
+ queueSize,
692
+ queryKey: query.queryKey,
693
+ queuePrefix: this.redisQueuePrefix,
694
+ requestId: query.requestId,
695
+ timeInQueue,
696
+ metadata: query.query?.metadata,
697
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
698
+ newVersionEntry: query.query?.newVersionEntry,
699
+ preAggregation: query.query?.preAggregation,
700
+ addedToQueueTime: query.addedToQueueTime,
701
+ });
702
+ await queueConnection.optimisticQueryUpdate(queryKeyHashed, { startQueryTime }, queueId);
703
+ let queryProcessHeartbeat = Date.now();
704
+ const heartBeatTimer = setInterval(async () => {
705
+ if ((Date.now() - queryProcessHeartbeat) > 5 * 60 * 1000) {
706
+ this.logger('Query processing heartbeat', {
740
707
  queueId,
741
- processingId,
742
- queueSize,
743
- duration: ((new Date()).getTime() - startQueryTime),
744
708
  queryKey: query.queryKey,
745
- queuePrefix: this.redisQueuePrefix,
746
709
  requestId: query.requestId,
747
- timeInQueue,
748
- metadata: query.query?.metadata,
749
- preAggregationId: query.query?.preAggregation?.preAggregationId,
750
- newVersionEntry: query.query?.newVersionEntry,
751
- preAggregation: query.query?.preAggregation,
752
- addedToQueueTime: query.addedToQueueTime,
753
710
  });
711
+ queryProcessHeartbeat = Date.now();
712
+ }
713
+ try {
714
+ await queueConnection.updateHeartBeat(queryKeyHashed, queueId);
754
715
  }
755
716
  catch (e) {
756
- executionResult = {
757
- error: (e.message || e).toString() // TODO error handling
758
- };
759
- this.logger('Error while querying', {
717
+ this.logger('Error updating heartbeat', {
760
718
  queueId,
761
- processingId,
762
- queueSize,
763
- duration: ((new Date()).getTime() - startQueryTime),
764
719
  queryKey: query.queryKey,
720
+ error: e.stack || e,
765
721
  queuePrefix: this.redisQueuePrefix,
766
722
  requestId: query.requestId,
767
- timeInQueue,
768
- metadata: query.query?.metadata,
769
- preAggregationId: query.query?.preAggregation?.preAggregationId,
770
- newVersionEntry: query.query?.newVersionEntry,
771
- preAggregation: query.query?.preAggregation,
772
- addedToQueueTime: query.addedToQueueTime,
773
- error: (e.stack || e).toString()
774
723
  });
775
- if (e instanceof TimeoutError_1.TimeoutError) {
776
- const queryWithCancelHandle = await queueConnection.getQueryDef(queryKeyHashed, queueId);
777
- if (queryWithCancelHandle) {
778
- this.logger('Cancelling query due to timeout', {
724
+ }
725
+ if (!queryExecutionFinished && localCancelHandler !== null) {
726
+ try {
727
+ const currentDef = await queueConnection.getQueryDef(queryKeyHashed, queueId);
728
+ if (!currentDef && !queryExecutionFinished) {
729
+ this.logger('Cancelling query due to external cancellation', {
779
730
  queueId,
780
- processingId,
781
- queryKey: queryWithCancelHandle.queryKey,
731
+ queryKey: query.queryKey,
782
732
  queuePrefix: this.redisQueuePrefix,
783
- requestId: queryWithCancelHandle.requestId,
784
- metadata: queryWithCancelHandle.query?.metadata,
785
- preAggregationId: queryWithCancelHandle.query?.preAggregation?.preAggregationId,
786
- newVersionEntry: queryWithCancelHandle.query?.newVersionEntry,
787
- preAggregation: queryWithCancelHandle.query?.preAggregation,
788
- addedToQueueTime: queryWithCancelHandle.addedToQueueTime,
733
+ requestId: query.requestId,
734
+ metadata: query.query?.metadata,
735
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
736
+ newVersionEntry: query.query?.newVersionEntry,
737
+ preAggregation: query.query?.preAggregation,
738
+ addedToQueueTime: query.addedToQueueTime,
789
739
  });
790
- await this.sendCancelMessageFn(queryWithCancelHandle, queueId);
740
+ const cancelQuery = { ...query, cancelHandler: localCancelHandler };
741
+ await this.processCancel(cancelQuery, queueId);
791
742
  }
792
743
  }
744
+ catch (e) {
745
+ this.logger('Error checking for external cancellation', {
746
+ queueId,
747
+ queryKey: query.queryKey,
748
+ error: e.stack || e,
749
+ queuePrefix: this.redisQueuePrefix,
750
+ requestId: query.requestId,
751
+ });
752
+ }
793
753
  }
794
- finally {
795
- queryExecutionFinished = true;
796
- clearInterval(heartBeatTimer);
754
+ }, this.heartBeatInterval * 1000);
755
+ try {
756
+ const handler = query?.queryHandler;
757
+ switch (handler) {
758
+ case 'stream':
759
+ // eslint-disable-next-line no-case-declarations
760
+ const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
761
+ try {
762
+ await this.streamHandler(query.query, queryStream);
763
+ // CubeStore has special handling for null
764
+ executionResult = {
765
+ streamResult: true
766
+ };
767
+ }
768
+ finally {
769
+ if (this.streams.get(queryKeyHashed) === queryStream) {
770
+ this.streams.delete(queryKeyHashed);
771
+ }
772
+ }
773
+ break;
774
+ default:
775
+ executionResult = {
776
+ result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
777
+ localCancelHandler = cancelHandler;
778
+ try {
779
+ await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, queueId);
780
+ }
781
+ catch (e) {
782
+ this.logger('Error while query update', {
783
+ queueId,
784
+ queryKey: query.queryKey,
785
+ error: e.stack || e,
786
+ queuePrefix: this.redisQueuePrefix,
787
+ requestId: query.requestId,
788
+ metadata: query.query?.metadata,
789
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
790
+ newVersionEntry: query.query?.newVersionEntry,
791
+ preAggregation: query.query?.preAggregation,
792
+ addedToQueueTime: query.addedToQueueTime,
793
+ });
794
+ }
795
+ }))
796
+ };
797
+ break;
797
798
  }
798
- if (!(await queueConnection.setResultAndRemoveQuery(queryKeyHashed, executionResult, processingId, queueId))) {
799
- this.logger('Orphaned execution result', {
800
- queueId,
801
- processingId,
802
- warn: 'Result for query was not set due to processing lock wasn\'t acquired',
803
- queryKey: query.queryKey,
804
- queuePrefix: this.redisQueuePrefix,
805
- requestId: query.requestId,
806
- metadata: query.query?.metadata,
807
- preAggregationId: query.query?.preAggregation?.preAggregationId,
808
- newVersionEntry: query.query?.newVersionEntry,
809
- preAggregation: query.query?.preAggregation,
810
- addedToQueueTime: query.addedToQueueTime,
811
- });
799
+ this.logger('Performing query completed', {
800
+ queueId,
801
+ queueSize,
802
+ duration: ((new Date()).getTime() - startQueryTime),
803
+ queryKey: query.queryKey,
804
+ queuePrefix: this.redisQueuePrefix,
805
+ requestId: query.requestId,
806
+ timeInQueue,
807
+ metadata: query.query?.metadata,
808
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
809
+ newVersionEntry: query.query?.newVersionEntry,
810
+ preAggregation: query.query?.preAggregation,
811
+ addedToQueueTime: query.addedToQueueTime,
812
+ });
813
+ }
814
+ catch (e) {
815
+ executionResult = {
816
+ error: (e.message || e).toString() // TODO error handling
817
+ };
818
+ this.logger('Error while querying', {
819
+ queueId,
820
+ queueSize,
821
+ duration: ((new Date()).getTime() - startQueryTime),
822
+ queryKey: query.queryKey,
823
+ queuePrefix: this.redisQueuePrefix,
824
+ requestId: query.requestId,
825
+ timeInQueue,
826
+ metadata: query.query?.metadata,
827
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
828
+ newVersionEntry: query.query?.newVersionEntry,
829
+ preAggregation: query.query?.preAggregation,
830
+ addedToQueueTime: query.addedToQueueTime,
831
+ error: (e.stack || e).toString()
832
+ });
833
+ if (e instanceof TimeoutError_1.TimeoutError) {
834
+ const queryWithCancelHandle = await queueConnection.getQueryDef(queryKeyHashed, queueId);
835
+ if (queryWithCancelHandle) {
836
+ this.logger('Cancelling query due to timeout', {
837
+ queueId,
838
+ queryKey: queryWithCancelHandle.queryKey,
839
+ queuePrefix: this.redisQueuePrefix,
840
+ requestId: queryWithCancelHandle.requestId,
841
+ metadata: queryWithCancelHandle.query?.metadata,
842
+ preAggregationId: queryWithCancelHandle.query?.preAggregation?.preAggregationId,
843
+ newVersionEntry: queryWithCancelHandle.query?.newVersionEntry,
844
+ preAggregation: queryWithCancelHandle.query?.preAggregation,
845
+ addedToQueueTime: queryWithCancelHandle.addedToQueueTime,
846
+ });
847
+ await this.sendCancelMessageFn(queryWithCancelHandle, queueId);
848
+ }
812
849
  }
813
- await this.reconcileQueue();
814
850
  }
815
- else {
816
- // TODO Ideally streaming queries should reconcile queue here after waiting on open slot however in practice continue wait timeout reconciles faster CPU-wise
817
- // if (query?.queryHandler === 'stream') {
818
- // const [active] = await queueConnection.getQueryStageState(true);
819
- // if (active && active.length > 0) {
820
- // await Promise.race(active.map(keyHash => queueConnection.getResultBlocking(keyHash)));
821
- // await this.reconcileQueue();
822
- // }
823
- // }
824
- this.logger('Skip processing', {
851
+ finally {
852
+ queryExecutionFinished = true;
853
+ clearInterval(heartBeatTimer);
854
+ }
855
+ if (!(await queueConnection.setResultAndRemoveQuery(queryKeyHashed, executionResult, queueId))) {
856
+ this.logger('Orphaned execution result', {
825
857
  queueId,
826
- processingId,
827
- queryKey: query && query.queryKey || queryKeyHashed,
828
- requestId: query && query.requestId,
858
+ warn: 'Result for query was not set because the queue item is no longer active',
859
+ queryKey: query.queryKey,
829
860
  queuePrefix: this.redisQueuePrefix,
830
- processingLockAcquired,
831
- query,
832
- insertedCount,
833
- activeKeys,
834
- activated,
835
- queryExists: !!query
861
+ requestId: query.requestId,
862
+ metadata: query.query?.metadata,
863
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
864
+ newVersionEntry: query.query?.newVersionEntry,
865
+ preAggregation: query.query?.preAggregation,
866
+ addedToQueueTime: query.addedToQueueTime,
836
867
  });
837
- await queueConnection.freeProcessingLock(queryKeyHashed, processingId, activated);
838
868
  }
869
+ await this.reconcileQueue();
839
870
  }
840
871
  catch (e) {
841
872
  this.logger('Queue storage error', {
842
873
  queueId,
843
- queryKey: query && query.queryKey || queryKeyHashed,
844
- requestId: query && query.requestId,
874
+ queryKey: query.queryKey,
875
+ requestId: query.requestId,
845
876
  error: (e.stack || e).toString(),
846
877
  queuePrefix: this.redisQueuePrefix
847
878
  });