@cubejs-backend/query-orchestrator 0.23.3 → 0.24.2

package/test/integration/QueryQueueRedis.test.js

@@ -1,6 +0,0 @@
-/* globals describe, test, expect, afterAll */
-const { QueryQueueTest } = require('../unit/QueryQueue.test');
-const RedisPool = require('../../orchestrator/RedisPool');
-
-QueryQueueTest('RedisPool', { cacheAndQueueDriver: 'redis', redisPool: new RedisPool() });
-QueryQueueTest('RedisNoPool', { cacheAndQueueDriver: 'redis', redisPool: new RedisPool({ poolMin: 0, poolMax: 0 }) });

package/test/unit/QueryOrchestrator.test.js

@@ -1,301 +0,0 @@
-/* globals describe, beforeEach, afterEach, test, expect */
-const QueryOrchestrator = require('../../orchestrator/QueryOrchestrator');
-
-class MockDriver {
-  constructor() {
-    this.tables = [];
-    this.tablesReady = [];
-    this.executedQueries = [];
-    this.cancelledQueries = [];
-  }
-
-  query(query) {
-    this.executedQueries.push(query);
-    let promise = Promise.resolve([query]);
-    if (query.match(`orders_too_big`)) {
-      promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 3000)));
-    }
-
-    if (query.match(`orders_delay`)) {
-      promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 800)));
-    }
-
-    if (this.tablesReady.find(t => query.indexOf(t) !== -1)) {
-      promise = promise.then(res => res.concat({ tableReady: true }));
-    }
-
-    promise.cancel = () => {
-      this.cancelledQueries.push(query);
-    };
-    return promise;
-  }
-
-  async getTablesQuery(schema) {
-    return this.tables.map(t => ({ table_name: t.replace(`${schema}.`, '') }));
-  }
-
-  async createSchemaIfNotExists(schema) {
-    this.schema = schema;
-    return null;
-  }
-
-  loadPreAggregationIntoTable(preAggregationTableName, loadSql) {
-    this.tables.push(preAggregationTableName.substring(0, 100));
-    const promise = this.query(loadSql);
-    const resPromise = promise.then(() => this.tablesReady.push(preAggregationTableName.substring(0, 100)));
-    resPromise.cancel = promise.cancel;
-    return resPromise;
-  }
-
-  async dropTable(tableName) {
-    this.tables = this.tables.filter(t => t !== tableName);
-    return this.query(`DROP TABLE ${tableName}`);
-  }
-}
-
-describe('QueryOrchestrator', () => {
-  let mockDriver = null;
-  const queryOrchestrator = new QueryOrchestrator(
-    'TEST',
-    async () => mockDriver,
-    (msg, params) => console.log(msg, params), {
-      preAggregationsOptions: {
-        queueOptions: {
-          executionTimeout: 1
-        },
-        usedTablePersistTime: 1
-      }
-    }
-  );
-
-  beforeEach(() => {
-    mockDriver = new MockDriver();
-  });
-
-  afterEach(async () => {
-    await queryOrchestrator.cleanup();
-  });
-
-  test('basic', async () => {
-    const query = {
-      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
-      values: ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_number_and_count20191101",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count20191101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"]],
-        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      }],
-      renewQuery: true,
-      requestId: 'basic'
-    };
-    const result = await queryOrchestrator.fetchQuery(query);
-    console.log(result.data[0]);
-    expect(result.data[0]).toMatch(/orders_number_and_count20191101_kjypcoio_5yftl5il/);
-  });
-
-  test('indexes', async () => {
-    const query = {
-      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
-      values: ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_number_and_count20191101",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count20191101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"]],
-        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]],
-        indexesSql: [{
-          sql: ['CREATE INDEX orders_number_and_count_week20191101 ON stb_pre_aggregations.orders_number_and_count20191101 ("orders__created_at_week")', []],
-          indexName: 'orders_number_and_count_week20191101'
-        }],
-      }],
-      renewQuery: true,
-      requestId: 'indexes'
-    };
-    const result = await queryOrchestrator.fetchQuery(query);
-    console.log(result.data[0]);
-    expect(result.data[0]).toMatch(/orders_number_and_count20191101_l3kvjcmu_khbemovd/);
-    expect(mockDriver.executedQueries.join(',')).toMatch(/CREATE INDEX orders_number_and_count_week20191101_l3kvjcmu_khbemovd/);
-  });
-
-  test('silent truncate', async () => {
-    const query = {
-      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
-      values: ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"]],
-        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]],
-      }],
-      renewQuery: true,
-      requestId: 'silent truncate'
-    };
-    let thrown = true;
-    try {
-      await queryOrchestrator.fetchQuery(query);
-      thrown = false;
-    } catch (e) {
-      expect(e.message).toMatch(/Pre-aggregation table is not found/);
-    }
-    expect(thrown).toBe(true);
-  });
-
-  test('cancel pre-aggregation', async () => {
-    const query = {
-      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20181101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
-      values: ["2018-11-01T00:00:00Z", "2018-11-30T23:59:59Z"],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_number_and_count20181101",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count20181101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders_too_big AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2018-11-01T00:00:00Z", "2018-11-30T23:59:59Z"]],
-        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      }],
-      renewQuery: true,
-      requestId: 'cancel pre-aggregation'
-    };
-    try {
-      await queryOrchestrator.fetchQuery(query);
-    } catch (e) {
-      expect(e.toString()).toMatch(/timeout/);
-    }
-    expect(mockDriver.cancelledQueries[0]).toMatch(/orders_too_big/);
-  });
-
-  test('save structure versions', async () => {
-    mockDriver.tables = [];
-    await queryOrchestrator.fetchQuery({
-      query: `SELECT * FROM stb_pre_aggregations.orders`,
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders AS SELECT * FROM public.orders", []],
-        invalidateKeyQueries: [["SELECT 1", []]]
-      }],
-      renewQuery: true,
-      requestId: 'save structure versions'
-    });
-
-    await queryOrchestrator.fetchQuery({
-      query: `SELECT * FROM stb_pre_aggregations.orders`,
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders AS SELECT * FROM public.orders1", []],
-        invalidateKeyQueries: [["SELECT 1", []]]
-      }],
-      renewQuery: true,
-      requestId: 'save structure versions'
-    });
-
-    await new Promise(resolve => setTimeout(() => resolve(), 1000));
-
-    for (let i = 0; i < 5; i++) {
-      await queryOrchestrator.fetchQuery({
-        query: `SELECT * FROM stb_pre_aggregations.orders`,
-        values: [],
-        cacheKeyQueries: {
-          renewalThreshold: 21600,
-          queries: []
-        },
-        preAggregations: [{
-          preAggregationsSchema: "stb_pre_aggregations",
-          tableName: "stb_pre_aggregations.orders",
-          loadSql: ["CREATE TABLE stb_pre_aggregations.orders AS SELECT * FROM public.orders", []],
-          invalidateKeyQueries: [["SELECT 2", []]]
-        }],
-        renewQuery: true,
-        requestId: 'save structure versions'
-      });
-    }
-    expect(mockDriver.tables).toContainEqual(expect.stringMatching(/orders_f5v4jw3p_4eysppzt/));
-    expect(mockDriver.tables).toContainEqual(expect.stringMatching(/orders_mjooke4_ezlvkhjl/));
-  });
-
-  test('intermittent empty rollup', async () => {
-    const firstQuery = queryOrchestrator.fetchQuery({
-      query: `SELECT * FROM stb_pre_aggregations.orders_d20181102`,
-      values: [],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: []
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_d20181102",
-        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_d20181102 AS SELECT * FROM public.orders_delay", []],
-        invalidateKeyQueries: [["SELECT 2", []]]
-      }],
-      requestId: 'intermittent empty rollup'
-    });
-
-    queryOrchestrator.fetchQuery({
-      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_d20181101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
-      values: ["2018-11-01T00:00:00Z", "2018-11-30T23:59:59Z"],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_d20181101",
-        loadSql: [
-          "CREATE TABLE stb_pre_aggregations.orders_d20181101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders_delay AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1",
-          ["2018-11-01T00:00:00Z", "2018-11-30T23:59:59Z"]
-        ],
-        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      }],
-      requestId: 'intermittent empty rollup'
-    });
-
-    await firstQuery;
-
-    const res = await queryOrchestrator.fetchQuery({
-      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_d20181101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
-      values: ["2018-11-01T00:00:00Z", "2018-11-30T23:59:59Z"],
-      cacheKeyQueries: {
-        renewalThreshold: 21600,
-        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      },
-      preAggregations: [{
-        preAggregationsSchema: "stb_pre_aggregations",
-        tableName: "stb_pre_aggregations.orders_d20181101",
-        loadSql: [
-          "CREATE TABLE stb_pre_aggregations.orders_d20181101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders_delay AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1",
-          ["2018-11-01T00:00:00Z", "2018-11-30T23:59:59Z"]
-        ],
-        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
-      }],
-      requestId: 'intermittent empty rollup'
-    });
-
-    console.log(res);
-
-    expect(res.data).toContainEqual(expect.objectContaining({ tableReady: true }));
-  });
-});

package/test/unit/QueryQueue.test.js

@@ -1,249 +0,0 @@
-/* globals describe, test, expect, afterAll */
-const QueryQueue = require('../../orchestrator/QueryQueue');
-
-const QueryQueueTest = (name, options) => {
-  describe(`QueryQueue${name}`, () => {
-    let delayCount = 0;
-    const delayFn = (result, delay) => new Promise(resolve => setTimeout(() => resolve(result), delay));
-    let cancelledQuery;
-    const queue = new QueryQueue('test_query_queue', {
-      queryHandlers: {
-        foo: async (query) => `${query[0]} bar`,
-        delay: async (query, setCancelHandler) => {
-          const result = query.result + delayCount;
-          delayCount += 1;
-          await setCancelHandler(result);
-          return delayFn(result, query.delay);
-        }
-      },
-      cancelHandlers: {
-        delay: (query) => {
-          console.log(`cancel call: ${JSON.stringify(query)}`);
-          cancelledQuery = query.queryKey;
-        }
-      },
-      continueWaitTimeout: 1,
-      executionTimeout: 2,
-      orphanedTimeout: 2,
-      concurrency: 1,
-      ...options
-    });
-
-    afterAll(async () => {
-      await options.redisPool.cleanup();
-    });
-
-    test('gutter', async () => {
-      const query = ['select * from'];
-      const result = await queue.executeInQueue('foo', query, query);
-      expect(result).toBe('select * from bar');
-    });
-
-    test('instant double wait resolve', async () => {
-      const results = await Promise.all([
-        queue.executeInQueue('delay', `instant`, { delay: 400, result: '2' }),
-        queue.executeInQueue('delay', `instant`, { delay: 400, result: '2' })
-      ]);
-      expect(results).toStrictEqual(['20', '20']);
-    });
-
-    test('priority', async () => {
-      delayCount = 0;
-      const result = await Promise.all([
-        queue.executeInQueue('delay', `11`, { delay: 600, result: '1' }, 1),
-        queue.executeInQueue('delay', `12`, { delay: 100, result: '2' }, 0),
-        queue.executeInQueue('delay', `13`, { delay: 100, result: '3' }, 10)
-      ]);
-      expect(parseInt(result.find(f => f[0] === '3'), 10) % 10).toBeLessThan(2);
-    });
-
-
-    test('timeout', async () => {
-      delayCount = 0;
-      const query = ['select * from 2'];
-      let errorString = '';
-      for (let i = 0; i < 5; i++) {
-        try {
-          await queue.executeInQueue('delay', query, { delay: 3000, result: '1' });
-          console.log(`Delay ${i}`);
-        } catch (e) {
-          if (e.message === 'Continue wait') {
-            // eslint-disable-next-line no-continue
-            continue;
-          }
-          errorString = e.toString();
-          break;
-        }
-      }
-      expect(errorString).toEqual(expect.stringContaining('timeout'));
-    });
-
-
-    test('stage reporting', async () => {
-      delayCount = 0;
-      const resultPromise = queue.executeInQueue('delay', '1', { delay: 200, result: '1' }, 0, { stageQueryKey: '1' });
-      await delayFn(null, 50);
-      expect((await queue.getQueryStage('1')).stage).toBe('Executing query');
-      await resultPromise;
-      expect(await queue.getQueryStage('1')).toEqual(undefined);
-    });
-
-    test('priority stage reporting', async () => {
-      delayCount = 0;
-      const resultPromise = queue.executeInQueue('delay', '31', { delay: 200, result: '1' }, 20, { stageQueryKey: '12' });
-      await delayFn(null, 50);
-      const resultPromise2 = queue.executeInQueue('delay', '32', { delay: 200, result: '1' }, 10, { stageQueryKey: '12' });
-      await delayFn(null, 50);
-      expect((await queue.getQueryStage('12', 10)).stage).toBe('#1 in queue');
-      await resultPromise;
-      await resultPromise2;
-      expect(await queue.getQueryStage('12')).toEqual(undefined);
-    });
-
-    test('negative priority', async () => {
-      delayCount = 0;
-      const results = [];
-      await Promise.all([
-        queue.executeInQueue('delay', '31', { delay: 400, result: '4' }, -10).then(r => results.push(r)),
-        queue.executeInQueue('delay', '32', { delay: 100, result: '3' }, -9).then(r => results.push(r)),
-        queue.executeInQueue('delay', '33', { delay: 100, result: '2' }, -8).then(r => results.push(r)),
-        queue.executeInQueue('delay', '34', { delay: 100, result: '1' }, -7).then(r => results.push(r))
-      ]);
-
-      results.splice(0, 1);
-
-      expect(results.map(r => parseInt(r[0], 10) - parseInt(results[0][0], 10))).toEqual([0, 1, 2]);
-    });
-
-    test('orphaned', async () => {
-      for (let i = 1; i <= 4; i++) {
-        await queue.executeInQueue('delay', `11${i}`, { delay: 50, result: `${i}` }, 0);
-      }
-      cancelledQuery = null;
-      delayCount = 0;
-
-      let result = queue.executeInQueue('delay', `111`, { delay: 800, result: '1' }, 0);
-      delayFn(null, 50).then(() => queue.executeInQueue('delay', `112`, { delay: 800, result: '2' }, 0)).catch(e => e);
-      delayFn(null, 60).then(() => queue.executeInQueue('delay', `113`, { delay: 500, result: '3' }, 0)).catch(e => e);
-      delayFn(null, 70).then(() => queue.executeInQueue('delay', `114`, { delay: 900, result: '4' }, 0)).catch(e => e);
-
-      expect(await result).toBe('10');
-      await queue.executeInQueue('delay', `112`, { delay: 800, result: '2' }, 0);
-      result = await queue.executeInQueue('delay', `113`, { delay: 900, result: '3' }, 0);
-      expect(result).toBe('32');
-      await delayFn(null, 200);
-      expect(cancelledQuery).toBe('114');
-      await queue.executeInQueue('delay', `114`, { delay: 50, result: '4' }, 0);
-    });
-
-    test('removed before reconciled', async () => {
-      const query = ['select * from'];
-      await queue.processQuery(query);
-      const result = await queue.executeInQueue('foo', query, query);
-      expect(result).toBe('select * from bar');
-    });
-
-    test('queue driver lock obtain race condition', async () => {
-      const redisClient = await queue.queueDriver.createConnection();
-      const redisClient2 = await queue.queueDriver.createConnection();
-      const priority = 10;
-      const time = new Date().getTime();
-      const keyScore = time + (10000 - priority) * 1E14;
-
-      // console.log(await redisClient.getQueryAndRemove('race'));
-      // console.log(await redisClient.getQueryAndRemove('race1'));
-
-      if (redisClient.redisClient) {
-        await redisClient2.redisClient.setAsync(redisClient.queryProcessingLockKey('race'), '100');
-        await redisClient.redisClient.watchAsync(redisClient.queryProcessingLockKey('race'));
-        await redisClient2.redisClient.setAsync(redisClient.queryProcessingLockKey('race'), Math.random());
-
-        const res = await redisClient.redisClient.multi()
-          .set(redisClient.queryProcessingLockKey('race'), '100')
-          .set(redisClient.queryProcessingLockKey('race1'), '100')
-          .execAsync();
-
-        expect(res).toBe(null);
-        await redisClient.redisClient.delAsync(redisClient.queryProcessingLockKey('race'));
-        await redisClient.redisClient.delAsync(redisClient.queryProcessingLockKey('race1'));
-      }
-
-      await queue.reconcileQueue();
-
-      await redisClient.addToQueue(
-        keyScore, 'race', time, 'handler', ['select'], priority, { stageQueryKey: 'race' }
-      );
-
-      await redisClient.addToQueue(
-        keyScore + 100, 'race2', time + 100, 'handler2', ['select2'], priority, { stageQueryKey: 'race2' }
-      );
-
-      const processingId1 = await redisClient.getNextProcessingId();
-      const processingId4 = await redisClient.getNextProcessingId();
-
-      await redisClient.freeProcessingLock('race', processingId1, true);
-      await redisClient.freeProcessingLock('race2', processingId4, true);
-
-      await redisClient2.retrieveForProcessing('race2', await redisClient.getNextProcessingId());
-
-      const processingId = await redisClient.getNextProcessingId();
-      const retrieve6 = await redisClient.retrieveForProcessing('race', processingId);
-      console.log(retrieve6);
-      expect(!!retrieve6[5]).toBe(true);
-
-      console.log(await redisClient.getQueryAndRemove('race'));
-      console.log(await redisClient.getQueryAndRemove('race2'));
-
-      await queue.queueDriver.release(redisClient);
-      await queue.queueDriver.release(redisClient2);
-    });
-
-    test('activated but lock is not acquired', async () => {
-      const redisClient = await queue.queueDriver.createConnection();
-      const redisClient2 = await queue.queueDriver.createConnection();
-      const priority = 10;
-      const time = new Date().getTime();
-      const keyScore = time + (10000 - priority) * 1E14;
-
-      await queue.reconcileQueue();
-
-      await redisClient.addToQueue(
-        keyScore, 'activated1', time, 'handler', ['select'], priority, { stageQueryKey: 'race' }
-      );
-
-      await redisClient.addToQueue(
-        keyScore + 100, 'activated2', time + 100, 'handler2', ['select2'], priority, { stageQueryKey: 'race2' }
-      );
-
-      const processingId1 = await redisClient.getNextProcessingId();
-      const processingId2 = await redisClient.getNextProcessingId();
-      const processingId3 = await redisClient.getNextProcessingId();
-
-      const retrieve1 = await redisClient.retrieveForProcessing('activated1', processingId1);
-      console.log(retrieve1);
-      const retrieve2 = await redisClient2.retrieveForProcessing('activated2', processingId2);
-      console.log(retrieve2);
-      console.log(await redisClient.freeProcessingLock('activated1', processingId1, retrieve1 && retrieve1[2].indexOf('activated1') !== -1));
-      const retrieve3 = await redisClient.retrieveForProcessing('activated2', processingId3);
-      console.log(retrieve3);
-      console.log(await redisClient.freeProcessingLock('activated2', processingId3, retrieve3 && retrieve3[2].indexOf('activated2') !== -1));
-      console.log(retrieve2[2].indexOf('activated2') !== -1);
-      console.log(await redisClient2.freeProcessingLock('activated2', processingId2, retrieve2 && retrieve2[2].indexOf('activated2') !== -1));
-
-      const retrieve4 = await redisClient.retrieveForProcessing('activated2', await redisClient.getNextProcessingId());
-      console.log(retrieve4);
-      expect(retrieve4[0]).toBe(1);
-      expect(!!retrieve4[5]).toBe(true);
-
-      console.log(await redisClient.getQueryAndRemove('activated1'));
-      console.log(await redisClient.getQueryAndRemove('activated2'));
-
-      await queue.queueDriver.release(redisClient);
-      await queue.queueDriver.release(redisClient2);
-    });
-  });
-};
-
-QueryQueueTest('Local');
-
-module.exports = QueryQueueTest;