@cubejs-backend/query-orchestrator 0.24.0 → 0.24.2

package/test/unit/QueryOrchestrator.test.js

@@ -1,373 +0,0 @@
-/* globals describe, beforeEach, afterEach, test, expect */
-const QueryOrchestrator = require('../../orchestrator/QueryOrchestrator');
-
-class MockDriver {
-  constructor() {
-    this.tables = [];
-    this.tablesReady = [];
-    this.executedQueries = [];
-    this.cancelledQueries = [];
-  }
-
-  query(query) {
-    this.executedQueries.push(query);
-    let promise = Promise.resolve([query]);
-    if (query.match('orders_too_big')) {
-      promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 3000)));
-    }
-
-    if (query.match('orders_delay')) {
-      promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 800)));
-    }
-
-    if (query.match(/^SELECT NOW\(\)$/)) {
-      promise = promise.then(() => new Date().toJSON());
-    }
-
-    if (this.tablesReady.find(t => query.indexOf(t) !== -1)) {
-      promise = promise.then(res => res.concat({ tableReady: true }));
-    }
-
-    promise.cancel = () => {
-      this.cancelledQueries.push(query);
-    };
-    return promise;
-  }
-
-  async getTablesQuery(schema) {
-    if (this.tablesQueryDelay) {
-      await this.delay(this.tablesQueryDelay);
-    }
-    return this.tables.map(t => ({ table_name: t.replace(`${schema}.`, '') }));
-  }
-
-  delay(timeout) {
-    return new Promise(resolve => setTimeout(() => resolve(), timeout));
-  }
-
-  async createSchemaIfNotExists(schema) {
-    this.schema = schema;
-    return null;
-  }
-
-  loadPreAggregationIntoTable(preAggregationTableName, loadSql) {
-    this.tables.push(preAggregationTableName.substring(0, 100));
-    const promise = this.query(loadSql);
-    const resPromise = promise.then(() => this.tablesReady.push(preAggregationTableName.substring(0, 100)));
-    resPromise.cancel = promise.cancel;
-    return resPromise;
-  }
-
-  async dropTable(tableName) {
-    this.tables = this.tables.filter(t => t !== tableName);
-    return this.query(`DROP TABLE ${tableName}`);
-  }
-}
-
-describe('QueryOrchestrator', () => {
-  let mockDriver = null;
-  const queryOrchestrator = new QueryOrchestrator(
-    'TEST',
-    async () => mockDriver,
-    (msg, params) => console.log(new Date().toJSON(), msg, params), {
-      preAggregationsOptions: {
-        queueOptions: {
-          executionTimeout: 1
-        },
-        usedTablePersistTime: 1
-      }
-    }
-  );
-
-  beforeEach(() => {
-    mockDriver = new MockDriver();
-  });
-
-  afterEach(async () => {
-    await queryOrchestrator.cleanup();
-  });
-
-  test('basic', async () => {
-    const query = {
-      query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union  WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000',
-      values: ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z'],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_number_and_count20191101',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders_number_and_count20191101 AS SELECT\n      date_trunc(\'week\', ("orders".created_at::timestamptz AT TIME ZONE \'UTC\')) "orders__created_at_week", count("orders".id) "orders__count", sum("orders".number) "orders__number"\n    FROM\n      public.orders AS "orders"\n  WHERE ("orders".created_at >= $1::timestamptz AND "orders".created_at <= $2::timestamptz) GROUP BY 1', ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z']],
-        invalidateKeyQueries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      }],
-      renewQuery: true,
-      requestId: 'basic'
-    };
-    const result = await queryOrchestrator.fetchQuery(query);
-    console.log(result.data[0]);
-    expect(result.data[0]).toMatch(/orders_number_and_count20191101_kjypcoio_5yftl5il/);
-  });
-
-  test('indexes', async () => {
-    const query = {
-      query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union  WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000',
-      values: ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z'],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_number_and_count20191101',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders_number_and_count20191101 AS SELECT\n      date_trunc(\'week\', ("orders".created_at::timestamptz AT TIME ZONE \'UTC\')) "orders__created_at_week", count("orders".id) "orders__count", sum("orders".number) "orders__number"\n    FROM\n      public.orders AS "orders"\n  WHERE ("orders".created_at >= $1::timestamptz AND "orders".created_at <= $2::timestamptz) GROUP BY 1', ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z']],
-        invalidateKeyQueries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]],
-        indexesSql: [{
-          sql: ['CREATE INDEX orders_number_and_count_week20191101 ON stb_pre_aggregations.orders_number_and_count20191101 ("orders__created_at_week")', []],
-          indexName: 'orders_number_and_count_week20191101'
-        }],
-      }],
-      renewQuery: true,
-      requestId: 'indexes'
-    };
-    const result = await queryOrchestrator.fetchQuery(query);
-    console.log(result.data[0]);
-    expect(result.data[0]).toMatch(/orders_number_and_count20191101_l3kvjcmu_khbemovd/);
-    expect(mockDriver.executedQueries.join(',')).toMatch(/CREATE INDEX orders_number_and_count_week20191101_l3kvjcmu_khbemovd/);
-  });
-
-  test('silent truncate', async () => {
-    const query = {
-      query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101) as partition_union  WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000',
-      values: ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z'],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101 AS SELECT\n      date_trunc(\'week\', ("orders".created_at::timestamptz AT TIME ZONE \'UTC\')) "orders__created_at_week", count("orders".id) "orders__count", sum("orders".number) "orders__number"\n    FROM\n      public.orders AS "orders"\n  WHERE ("orders".created_at >= $1::timestamptz AND "orders".created_at <= $2::timestamptz) GROUP BY 1', ['2019-11-01T00:00:00Z', '2019-11-30T23:59:59Z']],
-        invalidateKeyQueries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]],
-      }],
-      renewQuery: true,
-      requestId: 'silent truncate'
-    };
-    let thrown = true;
-    try {
-      await queryOrchestrator.fetchQuery(query);
-      thrown = false;
-    } catch (e) {
-      expect(e.message).toMatch(/Pre-aggregation table is not found/);
-    }
-    expect(thrown).toBe(true);
-  });
-
-  test('cancel pre-aggregation', async () => {
-    const query = {
-      query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20181101) as partition_union  WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000',
-      values: ['2018-11-01T00:00:00Z', '2018-11-30T23:59:59Z'],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_number_and_count20181101',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders_number_and_count20181101 AS SELECT\n      date_trunc(\'week\', ("orders".created_at::timestamptz AT TIME ZONE \'UTC\')) "orders__created_at_week", count("orders".id) "orders__count", sum("orders".number) "orders__number"\n    FROM\n      public.orders_too_big AS "orders"\n  WHERE ("orders".created_at >= $1::timestamptz AND "orders".created_at <= $2::timestamptz) GROUP BY 1', ['2018-11-01T00:00:00Z', '2018-11-30T23:59:59Z']],
-        invalidateKeyQueries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      }],
-      renewQuery: true,
-      requestId: 'cancel pre-aggregation'
-    };
-    try {
-      await queryOrchestrator.fetchQuery(query);
-    } catch (e) {
-      expect(e.toString()).toMatch(/timeout/);
-    }
-    expect(mockDriver.cancelledQueries[0]).toMatch(/orders_too_big/);
-  });
-
-  test('save structure versions', async () => {
-    mockDriver.tables = [];
-    await queryOrchestrator.fetchQuery({
-      query: 'SELECT * FROM stb_pre_aggregations.orders',
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders AS SELECT * FROM public.orders', []],
-        invalidateKeyQueries: [['SELECT 1', []]]
-      }],
-      renewQuery: true,
-      requestId: 'save structure versions'
-    });
-
-    await queryOrchestrator.fetchQuery({
-      query: 'SELECT * FROM stb_pre_aggregations.orders',
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders AS SELECT * FROM public.orders1', []],
-        invalidateKeyQueries: [['SELECT 1', []]]
-      }],
-      renewQuery: true,
-      requestId: 'save structure versions'
-    });
-
-    await new Promise(resolve => setTimeout(() => resolve(), 1000));
-
-    for (let i = 0; i < 5; i++) {
-      await queryOrchestrator.fetchQuery({
-        query: 'SELECT * FROM stb_pre_aggregations.orders',
-        values: [],
-        cacheKeyQueries: {
-          renewalThreshold: 21600,
-          queries: []
-        },
-        preAggregations: [{
-          preAggregationsSchema: 'stb_pre_aggregations',
-          tableName: 'stb_pre_aggregations.orders',
-          loadSql: ['CREATE TABLE stb_pre_aggregations.orders AS SELECT * FROM public.orders', []],
-          invalidateKeyQueries: [['SELECT 2', []]]
-        }],
-        renewQuery: true,
-        requestId: 'save structure versions'
-      });
-    }
-    expect(mockDriver.tables).toContainEqual(expect.stringMatching(/orders_f5v4jw3p_4eysppzt/));
-    expect(mockDriver.tables).toContainEqual(expect.stringMatching(/orders_mjooke4_ezlvkhjl/));
-  });
-
-  test('intermittent empty rollup', async () => {
-    const firstQuery = queryOrchestrator.fetchQuery({
-      query: 'SELECT * FROM stb_pre_aggregations.orders_d20181102',
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_d20181102',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders_d20181102 AS SELECT * FROM public.orders_delay', []],
-        invalidateKeyQueries: [['SELECT 2', []]]
-      }],
-      requestId: 'intermittent empty rollup'
-    });
-
-    queryOrchestrator.fetchQuery({
-      query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_d20181101) as partition_union  WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000',
-      values: ['2018-11-01T00:00:00Z', '2018-11-30T23:59:59Z'],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_d20181101',
-        loadSql: [
-          'CREATE TABLE stb_pre_aggregations.orders_d20181101 AS SELECT\n      date_trunc(\'week\', ("orders".created_at::timestamptz AT TIME ZONE \'UTC\')) "orders__created_at_week", count("orders".id) "orders__count", sum("orders".number) "orders__number"\n    FROM\n      public.orders_delay AS "orders"\n  WHERE ("orders".created_at >= $1::timestamptz AND "orders".created_at <= $2::timestamptz) GROUP BY 1',
-          ['2018-11-01T00:00:00Z', '2018-11-30T23:59:59Z']
-        ],
-        invalidateKeyQueries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      }],
-      requestId: 'intermittent empty rollup'
-    });
-
-    await firstQuery;
-
-    const res = await queryOrchestrator.fetchQuery({
-      query: 'SELECT "orders__created_at_week" "orders__created_at_week", sum("orders__count") "orders__count" FROM (SELECT * FROM stb_pre_aggregations.orders_d20181101) as partition_union  WHERE ("orders__created_at_week" >= ($1::timestamptz::timestamptz AT TIME ZONE \'UTC\') AND "orders__created_at_week" <= ($2::timestamptz::timestamptz AT TIME ZONE \'UTC\')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000',
-      values: ['2018-11-01T00:00:00Z', '2018-11-30T23:59:59Z'],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_d20181101',
-        loadSql: [
-          'CREATE TABLE stb_pre_aggregations.orders_d20181101 AS SELECT\n      date_trunc(\'week\', ("orders".created_at::timestamptz AT TIME ZONE \'UTC\')) "orders__created_at_week", count("orders".id) "orders__count", sum("orders".number) "orders__number"\n    FROM\n      public.orders_delay AS "orders"\n  WHERE ("orders".created_at >= $1::timestamptz AND "orders".created_at <= $2::timestamptz) GROUP BY 1',
-          ['2018-11-01T00:00:00Z', '2018-11-30T23:59:59Z']
-        ],
-        invalidateKeyQueries: [['SELECT date_trunc(\'hour\', (NOW()::timestamptz AT TIME ZONE \'UTC\')) as current_hour', []]]
-      }],
-      requestId: 'intermittent empty rollup'
-    });
-
-    console.log(res);
-
-    expect(res.data).toContainEqual(expect.objectContaining({ tableReady: true }));
-  });
-
-  test('continue serve old tables cache without resetting it', async () => {
-    mockDriver.tablesQueryDelay = 300;
-    const requestId = 'continue serve old tables cache without resetting it';
-    const baseQuery = {
-      query: 'SELECT * FROM stb_pre_aggregations.orders_d20181103',
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 1,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: 'stb_pre_aggregations',
-        tableName: 'stb_pre_aggregations.orders_d20181103',
-        loadSql: ['CREATE TABLE stb_pre_aggregations.orders_d20181103 AS SELECT * FROM public.orders', []],
-        invalidateKeyQueries: [['SELECT NOW()', []]],
-        refreshKeyRenewalThresholds: [0.001]
-      }]
-    };
-
-    // create from scratch
-    await queryOrchestrator.fetchQuery({
-      ...baseQuery,
-      requestId: `${requestId}: create from scratch`
-    });
-
-    // start renew refresh as scheduled refresh does
-    const refresh = queryOrchestrator.fetchQuery({
-      ...baseQuery,
-      renewQuery: true,
-      requestId: `${requestId}: start refresh`
-    });
-
-    await mockDriver.delay(100);
-
-    let firstResolve = null;
-
-    console.log('Starting race');
-
-    // If database has a significant delay for pre-aggregations tables fetch we should continue serve rollup cache
-    // instead of waiting tables fetch query to complete.
-    await Promise.all([
-      queryOrchestrator.fetchQuery({
-        ...baseQuery,
-        requestId: `${requestId}: race`
-      }).then(() => {
-        if (!firstResolve) {
-          firstResolve = 'query';
-        }
-      }),
-      mockDriver.delay(150).then(() => {
-        if (!firstResolve) {
-          firstResolve = 'delay';
-        }
-      })
-    ]);
-
-    await refresh;
-
-    expect(firstResolve).toBe('query');
-  });
-});

package/test/unit/QueryQueue.test.js

@@ -1,247 +0,0 @@
-/* globals describe, test, expect, afterAll */
-const QueryQueue = require('../../orchestrator/QueryQueue');
-
-const QueryQueueTest = (name, options) => {
-  describe(`QueryQueue${name}`, () => {
-    let delayCount = 0;
-    const delayFn = (result, delay) => new Promise(resolve => setTimeout(() => resolve(result), delay));
-    let cancelledQuery;
-    const queue = new QueryQueue('test_query_queue', {
-      queryHandlers: {
-        foo: async (query) => `${query[0]} bar`,
-        delay: async (query, setCancelHandler) => {
-          const result = query.result + delayCount;
-          delayCount += 1;
-          await setCancelHandler(result);
-          return delayFn(result, query.delay);
-        }
-      },
-      cancelHandlers: {
-        delay: (query) => {
-          console.log(`cancel call: ${JSON.stringify(query)}`);
-          cancelledQuery = query.queryKey;
-        }
-      },
-      continueWaitTimeout: 1,
-      executionTimeout: 2,
-      orphanedTimeout: 2,
-      concurrency: 1,
-      ...options
-    });
-
-    afterAll(async () => {
-      await options.redisPool.cleanup();
-    });
-
-    test('gutter', async () => {
-      const query = ['select * from'];
-      const result = await queue.executeInQueue('foo', query, query);
-      expect(result).toBe('select * from bar');
-    });
-
-    test('instant double wait resolve', async () => {
-      const results = await Promise.all([
-        queue.executeInQueue('delay', 'instant', { delay: 400, result: '2' }),
-        queue.executeInQueue('delay', 'instant', { delay: 400, result: '2' })
-      ]);
-      expect(results).toStrictEqual(['20', '20']);
-    });
-
-    test('priority', async () => {
-      delayCount = 0;
-      const result = await Promise.all([
-        queue.executeInQueue('delay', '11', { delay: 600, result: '1' }, 1),
-        queue.executeInQueue('delay', '12', { delay: 100, result: '2' }, 0),
-        queue.executeInQueue('delay', '13', { delay: 100, result: '3' }, 10)
-      ]);
-      expect(parseInt(result.find(f => f[0] === '3'), 10) % 10).toBeLessThan(2);
-    });
-
-    test('timeout', async () => {
-      delayCount = 0;
-      const query = ['select * from 2'];
-      let errorString = '';
-      for (let i = 0; i < 5; i++) {
-        try {
-          await queue.executeInQueue('delay', query, { delay: 3000, result: '1' });
-          console.log(`Delay ${i}`);
-        } catch (e) {
-          if (e.message === 'Continue wait') {
-            // eslint-disable-next-line no-continue
-            continue;
-          }
-          errorString = e.toString();
-          break;
-        }
-      }
-      expect(errorString).toEqual(expect.stringContaining('timeout'));
-    });
-
-    test('stage reporting', async () => {
-      delayCount = 0;
-      const resultPromise = queue.executeInQueue('delay', '1', { delay: 200, result: '1' }, 0, { stageQueryKey: '1' });
-      await delayFn(null, 50);
-      expect((await queue.getQueryStage('1')).stage).toBe('Executing query');
-      await resultPromise;
-      expect(await queue.getQueryStage('1')).toEqual(undefined);
-    });
-
-    test('priority stage reporting', async () => {
-      delayCount = 0;
-      const resultPromise = queue.executeInQueue('delay', '31', { delay: 200, result: '1' }, 20, { stageQueryKey: '12' });
-      await delayFn(null, 50);
-      const resultPromise2 = queue.executeInQueue('delay', '32', { delay: 200, result: '1' }, 10, { stageQueryKey: '12' });
-      await delayFn(null, 50);
-      expect((await queue.getQueryStage('12', 10)).stage).toBe('#1 in queue');
-      await resultPromise;
-      await resultPromise2;
-      expect(await queue.getQueryStage('12')).toEqual(undefined);
-    });
-
-    test('negative priority', async () => {
-      delayCount = 0;
-      const results = [];
-      await Promise.all([
-        queue.executeInQueue('delay', '31', { delay: 400, result: '4' }, -10).then(r => results.push(r)),
-        queue.executeInQueue('delay', '32', { delay: 100, result: '3' }, -9).then(r => results.push(r)),
-        queue.executeInQueue('delay', '33', { delay: 100, result: '2' }, -8).then(r => results.push(r)),
-        queue.executeInQueue('delay', '34', { delay: 100, result: '1' }, -7).then(r => results.push(r))
-      ]);
-
-      results.splice(0, 1);
-
-      expect(results.map(r => parseInt(r[0], 10) - parseInt(results[0][0], 10))).toEqual([0, 1, 2]);
-    });
-
-    test('orphaned', async () => {
-      for (let i = 1; i <= 4; i++) {
-        await queue.executeInQueue('delay', `11${i}`, { delay: 50, result: `${i}` }, 0);
-      }
-      cancelledQuery = null;
-      delayCount = 0;
-
-      let result = queue.executeInQueue('delay', '111', { delay: 800, result: '1' }, 0);
-      delayFn(null, 50).then(() => queue.executeInQueue('delay', '112', { delay: 800, result: '2' }, 0)).catch(e => e);
-      delayFn(null, 60).then(() => queue.executeInQueue('delay', '113', { delay: 500, result: '3' }, 0)).catch(e => e);
-      delayFn(null, 70).then(() => queue.executeInQueue('delay', '114', { delay: 900, result: '4' }, 0)).catch(e => e);
-
-      expect(await result).toBe('10');
-      await queue.executeInQueue('delay', '112', { delay: 800, result: '2' }, 0);
-      result = await queue.executeInQueue('delay', '113', { delay: 900, result: '3' }, 0);
-      expect(result).toBe('32');
-      await delayFn(null, 200);
-      expect(cancelledQuery).toBe('114');
-      await queue.executeInQueue('delay', '114', { delay: 50, result: '4' }, 0);
-    });
-
-    test('removed before reconciled', async () => {
-      const query = ['select * from'];
-      await queue.processQuery(query);
-      const result = await queue.executeInQueue('foo', query, query);
-      expect(result).toBe('select * from bar');
-    });
-
-    test('queue driver lock obtain race condition', async () => {
-      const redisClient = await queue.queueDriver.createConnection();
-      const redisClient2 = await queue.queueDriver.createConnection();
-      const priority = 10;
-      const time = new Date().getTime();
-      const keyScore = time + (10000 - priority) * 1E14;
-
-      // console.log(await redisClient.getQueryAndRemove('race'));
-      // console.log(await redisClient.getQueryAndRemove('race1'));
-
-      if (redisClient.redisClient) {
-        await redisClient2.redisClient.setAsync(redisClient.queryProcessingLockKey('race'), '100');
-        await redisClient.redisClient.watchAsync(redisClient.queryProcessingLockKey('race'));
-        await redisClient2.redisClient.setAsync(redisClient.queryProcessingLockKey('race'), Math.random());
-
-        const res = await redisClient.redisClient.multi()
-          .set(redisClient.queryProcessingLockKey('race'), '100')
-          .set(redisClient.queryProcessingLockKey('race1'), '100')
-          .execAsync();
-
-        expect(res).toBe(null);
-        await redisClient.redisClient.delAsync(redisClient.queryProcessingLockKey('race'));
-        await redisClient.redisClient.delAsync(redisClient.queryProcessingLockKey('race1'));
-      }
-
-      await queue.reconcileQueue();
-
-      await redisClient.addToQueue(
-        keyScore, 'race', time, 'handler', ['select'], priority, { stageQueryKey: 'race' }
-      );
-
-      await redisClient.addToQueue(
-        keyScore + 100, 'race2', time + 100, 'handler2', ['select2'], priority, { stageQueryKey: 'race2' }
-      );
-
-      const processingId1 = await redisClient.getNextProcessingId();
-      const processingId4 = await redisClient.getNextProcessingId();
-
-      await redisClient.freeProcessingLock('race', processingId1, true);
-      await redisClient.freeProcessingLock('race2', processingId4, true);
-
-      await redisClient2.retrieveForProcessing('race2', await redisClient.getNextProcessingId());
-
-      const processingId = await redisClient.getNextProcessingId();
-      const retrieve6 = await redisClient.retrieveForProcessing('race', processingId);
-      console.log(retrieve6);
-      expect(!!retrieve6[5]).toBe(true);
-
-      console.log(await redisClient.getQueryAndRemove('race'));
-      console.log(await redisClient.getQueryAndRemove('race2'));
-
-      await queue.queueDriver.release(redisClient);
-      await queue.queueDriver.release(redisClient2);
-    });
-
-    test('activated but lock is not acquired', async () => {
-      const redisClient = await queue.queueDriver.createConnection();
-      const redisClient2 = await queue.queueDriver.createConnection();
-      const priority = 10;
-      const time = new Date().getTime();
-      const keyScore = time + (10000 - priority) * 1E14;
-
-      await queue.reconcileQueue();
-
-      await redisClient.addToQueue(
-        keyScore, 'activated1', time, 'handler', ['select'], priority, { stageQueryKey: 'race' }
-      );
-
-      await redisClient.addToQueue(
-        keyScore + 100, 'activated2', time + 100, 'handler2', ['select2'], priority, { stageQueryKey: 'race2' }
-      );
-
-      const processingId1 = await redisClient.getNextProcessingId();
-      const processingId2 = await redisClient.getNextProcessingId();
-      const processingId3 = await redisClient.getNextProcessingId();
-
-      const retrieve1 = await redisClient.retrieveForProcessing('activated1', processingId1);
-      console.log(retrieve1);
-      const retrieve2 = await redisClient2.retrieveForProcessing('activated2', processingId2);
-      console.log(retrieve2);
-      console.log(await redisClient.freeProcessingLock('activated1', processingId1, retrieve1 && retrieve1[2].indexOf('activated1') !== -1));
-      const retrieve3 = await redisClient.retrieveForProcessing('activated2', processingId3);
-      console.log(retrieve3);
-      console.log(await redisClient.freeProcessingLock('activated2', processingId3, retrieve3 && retrieve3[2].indexOf('activated2') !== -1));
-      console.log(retrieve2[2].indexOf('activated2') !== -1);
-      console.log(await redisClient2.freeProcessingLock('activated2', processingId2, retrieve2 && retrieve2[2].indexOf('activated2') !== -1));
-
-      const retrieve4 = await redisClient.retrieveForProcessing('activated2', await redisClient.getNextProcessingId());
-      console.log(retrieve4);
-      expect(retrieve4[0]).toBe(1);
-      expect(!!retrieve4[5]).toBe(true);
-
-      console.log(await redisClient.getQueryAndRemove('activated1'));
-      console.log(await redisClient.getQueryAndRemove('activated2'));
-
-      await queue.queueDriver.release(redisClient);
-      await queue.queueDriver.release(redisClient2);
-    });
-  });
-};
-
-QueryQueueTest('Local');
-
-module.exports = QueryQueueTest;