@cubejs-backend/query-orchestrator 1.7.24 → 1.7.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.QueryQueue = void 0;
4
4
  const events_1 = require("events");
5
5
  const shared_1 = require("@cubejs-backend/shared");
6
+ const base_driver_1 = require("@cubejs-backend/base-driver");
6
7
  const cubestore_driver_1 = require("@cubejs-backend/cubestore-driver");
7
8
  const TimeoutError_1 = require("./TimeoutError");
8
9
  const ContinueWaitError_1 = require("./ContinueWaitError");
@@ -53,7 +54,7 @@ class QueryQueue {
53
54
  this.executionTimeout = options.executionTimeout || (0, shared_1.getEnv)('dbQueryTimeout');
54
55
  this.orphanedTimeout = options.orphanedTimeout || 120;
55
56
  this.heartBeatInterval = options.heartBeatInterval || 30;
56
- this.sendProcessMessageFn = options.sendProcessMessageFn || ((queryKey, queryId) => { this.processQuery(queryKey, queryId); });
57
+ this.sendProcessMessageFn = options.sendProcessMessageFn || ((retrieved) => { this.executeQuery(retrieved); });
57
58
  this.sendCancelMessageFn = options.sendCancelMessageFn || ((query, queueId) => { this.processCancel(query, queueId); });
58
59
  this.queryHandlers = options.queryHandlers;
59
60
  this.streamHandler = options.streamHandler;
@@ -107,7 +108,7 @@ class QueryQueue {
107
108
  *
108
109
  * @throw {ContinueWaitError}
109
110
  */
110
- async executeInQueue(queryHandler, queryKey, query, priority, executeOptions) {
111
+ async executeInQueue(queryHandler, queryKey, query, priority = base_driver_1.QueuePriority.Background, executeOptions) {
111
112
  const options = {
112
113
  queueId: this.generateQueueId(),
113
114
  ...executeOptions,
@@ -143,10 +144,8 @@ class QueryQueue {
143
144
  }
144
145
  const queueConnection = await this.queueDriver.createConnection();
145
146
  let waitingContext;
147
+ let streamWait = null;
146
148
  try {
147
- if (priority == null) {
148
- priority = 0;
149
- }
150
149
  if (!(priority >= -10000 && priority <= 10000)) {
151
150
  throw new Error('Priority should be between -10000 and 10000');
152
151
  }
@@ -163,12 +162,8 @@ class QueryQueue {
163
162
  if (jobExists)
164
163
  return null;
165
164
  }
166
- const time = new Date().getTime();
167
- const keyScore = time + (10000 - priority) * 1E14;
168
165
  options.orphanedTimeout = query.orphanedTimeout;
169
- const orphanedTimeout = 'orphanedTimeout' in query ? query.orphanedTimeout : this.orphanedTimeout;
170
- const orphanedTime = time + (orphanedTimeout * 1000);
171
- const [added, queueId, queueSize, addedToQueueTime] = await queueConnection.addToQueue(keyScore, queryKey, orphanedTime, queryHandler, query, priority, options);
166
+ const [added, queueId, queueSize, addedToQueueTime, retrieved] = await queueConnection.addToQueue(queryKey, queryHandler, query, priority, options);
172
167
  if (added > 0) {
173
168
  waitingContext = {
174
169
  queueId,
@@ -193,11 +188,23 @@ class QueryQueue {
193
188
  preAggregation: query.preAggregation,
194
189
  addedToQueueTime,
195
190
  persistent: !!queryKey.persistent,
191
+ fastTrack: !!retrieved,
196
192
  });
197
193
  }
198
- await this.reconcileQueue();
194
+ // Subscribing after the dispatch would lose the `streamStarted` event of a handler
195
+ // which starts fast
196
+ if (queryHandler === 'stream') {
197
+ streamWait = this.waitForQueryStream(queryKeyHash);
198
+ }
199
+ if (retrieved) {
200
+ // The item is active already, there is nothing for reconcile to pick up
201
+ await this.dispatchQuery(this.retrievedQuery(queryKeyHash, queueId, retrieved));
202
+ }
203
+ else {
204
+ await this.reconcileQueue();
205
+ }
199
206
  if (!added) {
200
- const queryDef = await queueConnection.getQueryDef(queryKeyHash, queueId);
207
+ const queryDef = retrieved ? retrieved[4] : await queueConnection.getQueryDef(queryKeyHash, queueId);
201
208
  if (queryDef) {
202
209
  waitingContext = {
203
210
  queueId,
@@ -209,44 +216,20 @@ class QueryQueue {
209
216
  };
210
217
  }
211
218
  }
212
- const [active, toProcess] = await queueConnection.getQueryStageState(true);
219
+ // A retrieval carries the active keys of its prefix, and a retrieved query is never pending,
220
+ // so it has no place in the queue to report
221
+ const [active, toProcess] = retrieved ? [retrieved[2], undefined] : await queueConnection.getQueryStageState(true);
213
222
  this.logger('Waiting for query', {
214
223
  ...waitingContext,
215
224
  queueSize,
216
225
  activeQueryKeys: active,
217
226
  toProcessQueryKeys: toProcess,
218
227
  active: active.indexOf(queryKeyHash) !== -1,
219
- queueIndex: toProcess.indexOf(queryKeyHash),
228
+ queueIndex: toProcess ? toProcess.indexOf(queryKeyHash) : -1,
229
+ fastTrack: !!retrieved,
220
230
  });
221
- // Stream processing goes here under assumption there's no way of a stream close just after it was added to the `streams` map.
222
- // Otherwise `streamStarted` event listener should go before the `reconcileQueue` call.
223
- // TODO: Fix an issue with a fast execution of stream handler which caused by removal of QueryStream from streams,
224
- // while EventListener doesnt start to listen for started stream event
225
- if (queryHandler === 'stream') {
226
- const self = this;
227
- result = await new Promise((resolve) => {
228
- let timeoutTimerId = null;
229
- const onStreamStarted = (streamStartedHash) => {
230
- if (streamStartedHash === queryKeyHash) {
231
- if (timeoutTimerId) {
232
- clearTimeout(timeoutTimerId);
233
- }
234
- resolve(self.getQueryStream(queryKeyHash));
235
- }
236
- };
237
- self.streamEvents.addListener('streamStarted', onStreamStarted);
238
- const stream = this.getQueryStream(queryKeyHash);
239
- if (stream) {
240
- self.streamEvents.removeListener('streamStarted', onStreamStarted);
241
- resolve(stream);
242
- }
243
- else {
244
- timeoutTimerId = setTimeout(() => {
245
- self.streamEvents.removeListener('streamStarted', onStreamStarted);
246
- resolve(null);
247
- }, this.continueWaitTimeout * 10000);
248
- }
249
- });
231
+ if (streamWait) {
232
+ result = await streamWait.promise;
250
233
  }
251
234
  else {
252
235
  // Result here won't be fetched for a jobed build query (initialized by
@@ -266,9 +249,58 @@ class QueryQueue {
266
249
  throw error;
267
250
  }
268
251
  finally {
252
+ streamWait?.dispose();
269
253
  this.queueDriver.release(queueConnection);
270
254
  }
271
255
  }
256
+ retrievedQuery(queryKeyHash, queueId, retrieved) {
257
+ const [, , , queueSize, query] = retrieved;
258
+ return {
259
+ queryKeyHash,
260
+ queueId,
261
+ processingId: queueId,
262
+ queueSize,
263
+ query,
264
+ };
265
+ }
266
+ /**
267
+ * `dispose` releases the listener and the timer, it's a no-op once the promise resolved.
268
+ */
269
+ waitForQueryStream(queryKeyHash) {
270
+ let timeoutTimerId = null;
271
+ let onStreamStarted = null;
272
+ const dispose = () => {
273
+ if (timeoutTimerId) {
274
+ clearTimeout(timeoutTimerId);
275
+ timeoutTimerId = null;
276
+ }
277
+ if (onStreamStarted) {
278
+ this.streamEvents.removeListener('streamStarted', onStreamStarted);
279
+ onStreamStarted = null;
280
+ }
281
+ };
282
+ const promise = new Promise((resolve) => {
283
+ onStreamStarted = (streamStartedHash) => {
284
+ if (streamStartedHash === queryKeyHash) {
285
+ dispose();
286
+ resolve(this.getQueryStream(queryKeyHash) ?? null);
287
+ }
288
+ };
289
+ this.streamEvents.addListener('streamStarted', onStreamStarted);
290
+ const stream = this.getQueryStream(queryKeyHash);
291
+ if (stream) {
292
+ dispose();
293
+ resolve(stream);
294
+ }
295
+ else {
296
+ timeoutTimerId = setTimeout(() => {
297
+ dispose();
298
+ resolve(null);
299
+ }, this.continueWaitTimeout * 10000);
300
+ }
301
+ });
302
+ return { promise, dispose };
303
+ }
272
304
  /**
273
305
  * @throw {Error}
274
306
  */
@@ -461,7 +493,8 @@ class QueryQueue {
461
493
  }
462
494
  })
463
495
  .slice(0, toProcessLimit)
464
- .map(([queryKey, queueId]) => this.sendProcessMessageFn(queryKey, queueId));
496
+ .map(([queryKey, queueId]) => this.processQuery(queryKey, queueId));
497
+ // Awaits the retrieval of every picked query, not their execution.
465
498
  await Promise.all(tasks);
466
499
  }
467
500
  finally {
@@ -592,10 +625,35 @@ class QueryQueue {
592
625
  return executionResult;
593
626
  }
594
627
  /**
595
- * Processing query specified by the `queryKey`. This method encapsulates most
596
- * of the logic related to the queue updates, heartbeat, etc.
628
+ * Retrieves the query specified by the `queryKeyHashed` and hands it over for execution.
597
629
  */
598
630
  async processQuery(queryKeyHashed, queueId) {
631
+ const retrieved = await this.retrieveQueryForProcessing(queryKeyHashed, queueId);
632
+ if (!retrieved) {
633
+ return;
634
+ }
635
+ await this.dispatchQuery(retrieved);
636
+ }
637
+ async dispatchQuery(retrieved) {
638
+ try {
639
+ await this.sendProcessMessageFn(retrieved);
640
+ }
641
+ catch (e) {
642
+ this.logger('Error while processing message', {
643
+ queueId: retrieved.queueId,
644
+ queryKey: retrieved.query.queryKey,
645
+ requestId: retrieved.query.requestId,
646
+ error: (e.stack || e).toString(),
647
+ queuePrefix: this.redisQueuePrefix
648
+ });
649
+ }
650
+ }
651
+ /**
652
+ * Acquires the processing lock for the query specified by the `queryKeyHashed` and moves it to
653
+ * the active set. Returns `null` when the retrieval didn't succeed, which means another node is
654
+ * already running the query or the concurrency budget is full.
655
+ */
656
+ async retrieveQueryForProcessing(queryKeyHashed, queueId) {
599
657
  const queueConnection = await this.queueDriver.createConnection();
600
658
  let insertedCount;
601
659
  let activeKeys;
@@ -617,231 +675,261 @@ class QueryQueue {
617
675
  if (!query) {
618
676
  query = await queueConnection.getQueryDef(queryKeyHashed, null);
619
677
  }
620
- if (query && insertedCount && activated && processingLockAcquired) {
621
- let executionResult;
622
- let queryExecutionFinished = false;
623
- // Set by the query handler's setCancelHandler callback once execution begins.
624
- // Not available on the original query def from retrieveForProcessing.
625
- let localCancelHandler = null;
626
- const startQueryTime = (new Date()).getTime();
627
- const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
628
- this.logger('Performing query', {
678
+ if (!query || !insertedCount || !activated || !processingLockAcquired) {
679
+ // TODO Ideally streaming queries should reconcile queue here after waiting on open slot however in practice continue wait timeout reconciles faster CPU-wise
680
+ // if (query?.queryHandler === 'stream') {
681
+ // const [active] = await queueConnection.getQueryStageState(true);
682
+ // if (active && active.length > 0) {
683
+ // await Promise.race(active.map(keyHash => queueConnection.getResultBlocking(keyHash)));
684
+ // await this.reconcileQueue();
685
+ // }
686
+ // }
687
+ this.logger('Skip processing', {
629
688
  queueId,
630
689
  processingId,
631
- queueSize,
632
- queryKey: query.queryKey,
690
+ queryKey: query && query.queryKey || queryKeyHashed,
691
+ requestId: query && query.requestId,
633
692
  queuePrefix: this.redisQueuePrefix,
634
- requestId: query.requestId,
635
- timeInQueue,
636
- metadata: query.query?.metadata,
637
- preAggregationId: query.query?.preAggregation?.preAggregationId,
638
- newVersionEntry: query.query?.newVersionEntry,
639
- preAggregation: query.query?.preAggregation,
640
- addedToQueueTime: query.addedToQueueTime,
693
+ processingLockAcquired,
694
+ query,
695
+ insertedCount,
696
+ activeKeys,
697
+ activated,
698
+ queryExists: !!query
641
699
  });
642
- await queueConnection.optimisticQueryUpdate(queryKeyHashed, { startQueryTime }, processingId, queueId);
643
- let queryProcessHeartbeat = Date.now();
644
- const heartBeatTimer = setInterval(async () => {
645
- if ((Date.now() - queryProcessHeartbeat) > 5 * 60 * 1000) {
646
- this.logger('Query processing heartbeat', {
647
- queueId,
648
- queryKey: query.queryKey,
649
- requestId: query.requestId,
650
- });
651
- queryProcessHeartbeat = Date.now();
652
- }
653
- try {
654
- await queueConnection.updateHeartBeat(queryKeyHashed, queueId);
655
- }
656
- catch (e) {
657
- this.logger('Error updating heartbeat', {
658
- queueId,
659
- queryKey: query.queryKey,
660
- error: e.stack || e,
661
- queuePrefix: this.redisQueuePrefix,
662
- requestId: query.requestId,
663
- });
664
- }
665
- if (!queryExecutionFinished && localCancelHandler !== null) {
666
- try {
667
- const currentDef = await queueConnection.getQueryDef(queryKeyHashed, queueId);
668
- if (!currentDef && !queryExecutionFinished) {
669
- this.logger('Cancelling query due to external cancellation', {
670
- queueId,
671
- queryKey: query.queryKey,
672
- queuePrefix: this.redisQueuePrefix,
673
- requestId: query.requestId,
674
- metadata: query.query?.metadata,
675
- preAggregationId: query.query?.preAggregation?.preAggregationId,
676
- newVersionEntry: query.query?.newVersionEntry,
677
- preAggregation: query.query?.preAggregation,
678
- addedToQueueTime: query.addedToQueueTime,
679
- });
680
- const cancelQuery = { ...query, cancelHandler: localCancelHandler };
681
- await this.processCancel(cancelQuery, queueId);
682
- }
683
- }
684
- catch (e) {
685
- this.logger('Error checking for external cancellation', {
686
- queueId,
687
- queryKey: query.queryKey,
688
- error: e.stack || e,
689
- queuePrefix: this.redisQueuePrefix,
690
- requestId: query.requestId,
691
- });
692
- }
693
- }
694
- }, this.heartBeatInterval * 1000);
695
- try {
696
- const handler = query?.queryHandler;
697
- switch (handler) {
698
- case 'stream':
699
- // eslint-disable-next-line no-case-declarations
700
- const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
701
- try {
702
- await this.streamHandler(query.query, queryStream);
703
- // CubeStore has special handling for null
704
- executionResult = {
705
- streamResult: true
706
- };
707
- }
708
- finally {
709
- if (this.streams.get(queryKeyHashed) === queryStream) {
710
- this.streams.delete(queryKeyHashed);
711
- }
712
- }
713
- break;
714
- default:
715
- executionResult = {
716
- result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
717
- localCancelHandler = cancelHandler;
718
- try {
719
- await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, processingId, queueId);
720
- }
721
- catch (e) {
722
- this.logger('Error while query update', {
723
- queueId,
724
- queryKey: query.queryKey,
725
- error: e.stack || e,
726
- queuePrefix: this.redisQueuePrefix,
727
- requestId: query.requestId,
728
- metadata: query.query?.metadata,
729
- preAggregationId: query.query?.preAggregation?.preAggregationId,
730
- newVersionEntry: query.query?.newVersionEntry,
731
- preAggregation: query.query?.preAggregation,
732
- addedToQueueTime: query.addedToQueueTime,
733
- });
734
- }
735
- }))
736
- };
737
- break;
738
- }
739
- this.logger('Performing query completed', {
700
+ await queueConnection.freeProcessingLock(queryKeyHashed, processingId, activated);
701
+ return null;
702
+ }
703
+ return {
704
+ queryKeyHash: queryKeyHashed,
705
+ queueId,
706
+ processingId,
707
+ queueSize,
708
+ query,
709
+ };
710
+ }
711
+ catch (e) {
712
+ this.logger('Queue storage error', {
713
+ queueId,
714
+ queryKey: query && query.queryKey || queryKeyHashed,
715
+ requestId: query && query.requestId,
716
+ error: (e.stack || e).toString(),
717
+ queuePrefix: this.redisQueuePrefix
718
+ });
719
+ return null;
720
+ }
721
+ finally {
722
+ this.queueDriver.release(queueConnection);
723
+ }
724
+ }
725
+ /**
726
+ * Executes a retrieved query: runs its handler while keeping the queue heartbeat alive, then acks
727
+ * the result. It's the counterpart of `sendProcessMessageFn` and the entry point for a custom
728
+ * implementation which hands the query over to another process.
729
+ */
730
+ async executeQuery(retrieved) {
731
+ const { queryKeyHash: queryKeyHashed, queueId, processingId, queueSize, query } = retrieved;
732
+ const queueConnection = await this.queueDriver.createConnection();
733
+ try {
734
+ let executionResult;
735
+ let queryExecutionFinished = false;
736
+ // Set by the query handler's setCancelHandler callback once execution begins.
737
+ // Not available on the original query def from retrieveForProcessing.
738
+ let localCancelHandler = null;
739
+ const startQueryTime = (new Date()).getTime();
740
+ const timeInQueue = (new Date()).getTime() - query.addedToQueueTime;
741
+ this.logger('Performing query', {
742
+ queueId,
743
+ processingId,
744
+ queueSize,
745
+ queryKey: query.queryKey,
746
+ queuePrefix: this.redisQueuePrefix,
747
+ requestId: query.requestId,
748
+ timeInQueue,
749
+ metadata: query.query?.metadata,
750
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
751
+ newVersionEntry: query.query?.newVersionEntry,
752
+ preAggregation: query.query?.preAggregation,
753
+ addedToQueueTime: query.addedToQueueTime,
754
+ });
755
+ await queueConnection.optimisticQueryUpdate(queryKeyHashed, { startQueryTime }, processingId, queueId);
756
+ let queryProcessHeartbeat = Date.now();
757
+ const heartBeatTimer = setInterval(async () => {
758
+ if ((Date.now() - queryProcessHeartbeat) > 5 * 60 * 1000) {
759
+ this.logger('Query processing heartbeat', {
740
760
  queueId,
741
- processingId,
742
- queueSize,
743
- duration: ((new Date()).getTime() - startQueryTime),
744
761
  queryKey: query.queryKey,
745
- queuePrefix: this.redisQueuePrefix,
746
762
  requestId: query.requestId,
747
- timeInQueue,
748
- metadata: query.query?.metadata,
749
- preAggregationId: query.query?.preAggregation?.preAggregationId,
750
- newVersionEntry: query.query?.newVersionEntry,
751
- preAggregation: query.query?.preAggregation,
752
- addedToQueueTime: query.addedToQueueTime,
753
763
  });
764
+ queryProcessHeartbeat = Date.now();
765
+ }
766
+ try {
767
+ await queueConnection.updateHeartBeat(queryKeyHashed, queueId);
754
768
  }
755
769
  catch (e) {
756
- executionResult = {
757
- error: (e.message || e).toString() // TODO error handling
758
- };
759
- this.logger('Error while querying', {
770
+ this.logger('Error updating heartbeat', {
760
771
  queueId,
761
- processingId,
762
- queueSize,
763
- duration: ((new Date()).getTime() - startQueryTime),
764
772
  queryKey: query.queryKey,
773
+ error: e.stack || e,
765
774
  queuePrefix: this.redisQueuePrefix,
766
775
  requestId: query.requestId,
767
- timeInQueue,
768
- metadata: query.query?.metadata,
769
- preAggregationId: query.query?.preAggregation?.preAggregationId,
770
- newVersionEntry: query.query?.newVersionEntry,
771
- preAggregation: query.query?.preAggregation,
772
- addedToQueueTime: query.addedToQueueTime,
773
- error: (e.stack || e).toString()
774
776
  });
775
- if (e instanceof TimeoutError_1.TimeoutError) {
776
- const queryWithCancelHandle = await queueConnection.getQueryDef(queryKeyHashed, queueId);
777
- if (queryWithCancelHandle) {
778
- this.logger('Cancelling query due to timeout', {
777
+ }
778
+ if (!queryExecutionFinished && localCancelHandler !== null) {
779
+ try {
780
+ const currentDef = await queueConnection.getQueryDef(queryKeyHashed, queueId);
781
+ if (!currentDef && !queryExecutionFinished) {
782
+ this.logger('Cancelling query due to external cancellation', {
779
783
  queueId,
780
- processingId,
781
- queryKey: queryWithCancelHandle.queryKey,
784
+ queryKey: query.queryKey,
782
785
  queuePrefix: this.redisQueuePrefix,
783
- requestId: queryWithCancelHandle.requestId,
784
- metadata: queryWithCancelHandle.query?.metadata,
785
- preAggregationId: queryWithCancelHandle.query?.preAggregation?.preAggregationId,
786
- newVersionEntry: queryWithCancelHandle.query?.newVersionEntry,
787
- preAggregation: queryWithCancelHandle.query?.preAggregation,
788
- addedToQueueTime: queryWithCancelHandle.addedToQueueTime,
786
+ requestId: query.requestId,
787
+ metadata: query.query?.metadata,
788
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
789
+ newVersionEntry: query.query?.newVersionEntry,
790
+ preAggregation: query.query?.preAggregation,
791
+ addedToQueueTime: query.addedToQueueTime,
789
792
  });
790
- await this.sendCancelMessageFn(queryWithCancelHandle, queueId);
793
+ const cancelQuery = { ...query, cancelHandler: localCancelHandler };
794
+ await this.processCancel(cancelQuery, queueId);
791
795
  }
792
796
  }
797
+ catch (e) {
798
+ this.logger('Error checking for external cancellation', {
799
+ queueId,
800
+ queryKey: query.queryKey,
801
+ error: e.stack || e,
802
+ queuePrefix: this.redisQueuePrefix,
803
+ requestId: query.requestId,
804
+ });
805
+ }
793
806
  }
794
- finally {
795
- queryExecutionFinished = true;
796
- clearInterval(heartBeatTimer);
807
+ }, this.heartBeatInterval * 1000);
808
+ try {
809
+ const handler = query?.queryHandler;
810
+ switch (handler) {
811
+ case 'stream':
812
+ // eslint-disable-next-line no-case-declarations
813
+ const queryStream = this.createQueryStream(queryKeyHashed, query.query?.aliasNameToMember);
814
+ try {
815
+ await this.streamHandler(query.query, queryStream);
816
+ // CubeStore has special handling for null
817
+ executionResult = {
818
+ streamResult: true
819
+ };
820
+ }
821
+ finally {
822
+ if (this.streams.get(queryKeyHashed) === queryStream) {
823
+ this.streams.delete(queryKeyHashed);
824
+ }
825
+ }
826
+ break;
827
+ default:
828
+ executionResult = {
829
+ result: await this.queryTimeout(this.queryHandlers[handler](query.query, async (cancelHandler) => {
830
+ localCancelHandler = cancelHandler;
831
+ try {
832
+ await queueConnection.optimisticQueryUpdate(queryKeyHashed, { cancelHandler }, processingId, queueId);
833
+ }
834
+ catch (e) {
835
+ this.logger('Error while query update', {
836
+ queueId,
837
+ queryKey: query.queryKey,
838
+ error: e.stack || e,
839
+ queuePrefix: this.redisQueuePrefix,
840
+ requestId: query.requestId,
841
+ metadata: query.query?.metadata,
842
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
843
+ newVersionEntry: query.query?.newVersionEntry,
844
+ preAggregation: query.query?.preAggregation,
845
+ addedToQueueTime: query.addedToQueueTime,
846
+ });
847
+ }
848
+ }))
849
+ };
850
+ break;
797
851
  }
798
- if (!(await queueConnection.setResultAndRemoveQuery(queryKeyHashed, executionResult, processingId, queueId))) {
799
- this.logger('Orphaned execution result', {
800
- queueId,
801
- processingId,
802
- warn: 'Result for query was not set due to processing lock wasn\'t acquired',
803
- queryKey: query.queryKey,
804
- queuePrefix: this.redisQueuePrefix,
805
- requestId: query.requestId,
806
- metadata: query.query?.metadata,
807
- preAggregationId: query.query?.preAggregation?.preAggregationId,
808
- newVersionEntry: query.query?.newVersionEntry,
809
- preAggregation: query.query?.preAggregation,
810
- addedToQueueTime: query.addedToQueueTime,
811
- });
852
+ this.logger('Performing query completed', {
853
+ queueId,
854
+ processingId,
855
+ queueSize,
856
+ duration: ((new Date()).getTime() - startQueryTime),
857
+ queryKey: query.queryKey,
858
+ queuePrefix: this.redisQueuePrefix,
859
+ requestId: query.requestId,
860
+ timeInQueue,
861
+ metadata: query.query?.metadata,
862
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
863
+ newVersionEntry: query.query?.newVersionEntry,
864
+ preAggregation: query.query?.preAggregation,
865
+ addedToQueueTime: query.addedToQueueTime,
866
+ });
867
+ }
868
+ catch (e) {
869
+ executionResult = {
870
+ error: (e.message || e).toString() // TODO error handling
871
+ };
872
+ this.logger('Error while querying', {
873
+ queueId,
874
+ processingId,
875
+ queueSize,
876
+ duration: ((new Date()).getTime() - startQueryTime),
877
+ queryKey: query.queryKey,
878
+ queuePrefix: this.redisQueuePrefix,
879
+ requestId: query.requestId,
880
+ timeInQueue,
881
+ metadata: query.query?.metadata,
882
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
883
+ newVersionEntry: query.query?.newVersionEntry,
884
+ preAggregation: query.query?.preAggregation,
885
+ addedToQueueTime: query.addedToQueueTime,
886
+ error: (e.stack || e).toString()
887
+ });
888
+ if (e instanceof TimeoutError_1.TimeoutError) {
889
+ const queryWithCancelHandle = await queueConnection.getQueryDef(queryKeyHashed, queueId);
890
+ if (queryWithCancelHandle) {
891
+ this.logger('Cancelling query due to timeout', {
892
+ queueId,
893
+ processingId,
894
+ queryKey: queryWithCancelHandle.queryKey,
895
+ queuePrefix: this.redisQueuePrefix,
896
+ requestId: queryWithCancelHandle.requestId,
897
+ metadata: queryWithCancelHandle.query?.metadata,
898
+ preAggregationId: queryWithCancelHandle.query?.preAggregation?.preAggregationId,
899
+ newVersionEntry: queryWithCancelHandle.query?.newVersionEntry,
900
+ preAggregation: queryWithCancelHandle.query?.preAggregation,
901
+ addedToQueueTime: queryWithCancelHandle.addedToQueueTime,
902
+ });
903
+ await this.sendCancelMessageFn(queryWithCancelHandle, queueId);
904
+ }
812
905
  }
813
- await this.reconcileQueue();
814
906
  }
815
- else {
816
- // TODO Ideally streaming queries should reconcile queue here after waiting on open slot however in practice continue wait timeout reconciles faster CPU-wise
817
- // if (query?.queryHandler === 'stream') {
818
- // const [active] = await queueConnection.getQueryStageState(true);
819
- // if (active && active.length > 0) {
820
- // await Promise.race(active.map(keyHash => queueConnection.getResultBlocking(keyHash)));
821
- // await this.reconcileQueue();
822
- // }
823
- // }
824
- this.logger('Skip processing', {
907
+ finally {
908
+ queryExecutionFinished = true;
909
+ clearInterval(heartBeatTimer);
910
+ }
911
+ if (!(await queueConnection.setResultAndRemoveQuery(queryKeyHashed, executionResult, processingId, queueId))) {
912
+ this.logger('Orphaned execution result', {
825
913
  queueId,
826
914
  processingId,
827
- queryKey: query && query.queryKey || queryKeyHashed,
828
- requestId: query && query.requestId,
915
+ warn: 'Result for query was not set due to processing lock wasn\'t acquired',
916
+ queryKey: query.queryKey,
829
917
  queuePrefix: this.redisQueuePrefix,
830
- processingLockAcquired,
831
- query,
832
- insertedCount,
833
- activeKeys,
834
- activated,
835
- queryExists: !!query
918
+ requestId: query.requestId,
919
+ metadata: query.query?.metadata,
920
+ preAggregationId: query.query?.preAggregation?.preAggregationId,
921
+ newVersionEntry: query.query?.newVersionEntry,
922
+ preAggregation: query.query?.preAggregation,
923
+ addedToQueueTime: query.addedToQueueTime,
836
924
  });
837
- await queueConnection.freeProcessingLock(queryKeyHashed, processingId, activated);
838
925
  }
926
+ await this.reconcileQueue();
839
927
  }
840
928
  catch (e) {
841
929
  this.logger('Queue storage error', {
842
930
  queueId,
843
- queryKey: query && query.queryKey || queryKeyHashed,
844
- requestId: query && query.requestId,
931
+ queryKey: query.queryKey,
932
+ requestId: query.requestId,
845
933
  error: (e.stack || e).toString(),
846
934
  queuePrefix: this.redisQueuePrefix
847
935
  });