@cubejs-backend/query-orchestrator 0.24.0 → 0.24.2

package/orchestrator/RedisQueueDriver.js

@@ -1,286 +1,8 @@
-const R = require('ramda');
+const { RedisQueueDriver } = require('../dist/src/orchestrator/RedisQueueDriver');
 
-const BaseQueueDriver = require('./BaseQueueDriver');
-
-class RedisQueueDriverConnection {
-  constructor(driver, options) {
-    this.driver = driver;
-    this.redisClient = options.redisClient;
-    this.redisQueuePrefix = options.redisQueuePrefix;
-    this.continueWaitTimeout = options.continueWaitTimeout;
-    this.orphanedTimeout = options.orphanedTimeout;
-    this.heartBeatTimeout = options.heartBeatTimeout;
-    this.concurrency = options.concurrency;
-  }
-
-  async getResultBlocking(queryKey) {
-    const resultListKey = this.resultListKey(queryKey);
-    if (!(await this.redisClient.hgetAsync([this.queriesDefKey(), this.redisHash(queryKey)]))) {
-      return this.getResult(queryKey);
-    }
-    const result = await this.redisClient.brpopAsync([resultListKey, this.continueWaitTimeout]);
-    if (result) {
-      await this.redisClient.lpushAsync([resultListKey, result[1]]);
-      await this.redisClient.rpopAsync(resultListKey);
-    }
-    return result && JSON.parse(result[1]);
-  }
-
-  async getResult(queryKey) {
-    const resultListKey = this.resultListKey(queryKey);
-    const result = await this.redisClient.rpopAsync(resultListKey);
-    return result && JSON.parse(result);
-  }
-
-  addToQueue(keyScore, queryKey, time, queryHandler, query, priority, options) {
-    return this.redisClient.multi()
-      .zadd([this.toProcessRedisKey(), 'NX', keyScore, this.redisHash(queryKey)])
-      .zadd([this.recentRedisKey(), time, this.redisHash(queryKey)])
-      .hsetnx([
-        this.queriesDefKey(),
-        this.redisHash(queryKey),
-        JSON.stringify({
-          queryHandler,
-          query,
-          queryKey,
-          stageQueryKey: options.stageQueryKey,
-          priority,
-          requestId: options.requestId,
-          addedToQueueTime: new Date().getTime()
-        })
-      ])
-      .zcard(this.toProcessRedisKey())
-      .execAsync();
-  }
-
-  getToProcessQueries() {
-    return this.redisClient.zrangeAsync([this.toProcessRedisKey(), 0, -1]);
-  }
-
-  getActiveQueries() {
-    return this.redisClient.zrangeAsync([this.activeRedisKey(), 0, -1]);
-  }
-
-  async getQueryAndRemove(queryKey) {
-    const [query, ...restResult] = await this.redisClient.multi()
-      .hget([this.queriesDefKey(), this.redisHash(queryKey)])
-      .zrem([this.activeRedisKey(), this.redisHash(queryKey)])
-      .zrem([this.heartBeatRedisKey(), this.redisHash(queryKey)])
-      .zrem([this.toProcessRedisKey(), this.redisHash(queryKey)])
-      .zrem([this.recentRedisKey(), this.redisHash(queryKey)])
-      .hdel([this.queriesDefKey(), this.redisHash(queryKey)])
-      .del(this.queryProcessingLockKey(queryKey))
-      .execAsync();
-    return [JSON.parse(query), ...restResult];
-  }
-
-  async setResultAndRemoveQuery(queryKey, executionResult, processingId) {
-    try {
-      await this.redisClient.watchAsync(this.queryProcessingLockKey(queryKey));
-      const currentProcessId = await this.redisClient.getAsync(this.queryProcessingLockKey(queryKey));
-      if (processingId !== currentProcessId) {
-        return false;
-      }
-
-      return this.redisClient.multi()
-        .lpush([this.resultListKey(queryKey), JSON.stringify(executionResult)])
-        .zrem([this.activeRedisKey(), this.redisHash(queryKey)])
-        .zrem([this.heartBeatRedisKey(), this.redisHash(queryKey)])
-        .zrem([this.toProcessRedisKey(), this.redisHash(queryKey)])
-        .zrem([this.recentRedisKey(), this.redisHash(queryKey)])
-        .hdel([this.queriesDefKey(), this.redisHash(queryKey)])
-        .del(this.queryProcessingLockKey(queryKey))
-        .execAsync();
-    } finally {
-      await this.redisClient.unwatchAsync();
-    }
-  }
-
-  getOrphanedQueries() {
-    return this.redisClient.zrangebyscoreAsync(
-      [this.recentRedisKey(), 0, (new Date().getTime() - this.orphanedTimeout * 1000)]
-    );
-  }
-
-  getStalledQueries() {
-    return this.redisClient.zrangebyscoreAsync(
-      [this.heartBeatRedisKey(), 0, (new Date().getTime() - this.heartBeatTimeout * 1000)]
-    );
-  }
-
-  async getQueryStageState(onlyKeys) {
-    let request = this.redisClient.multi()
-      .zrange([this.activeRedisKey(), 0, -1])
-      .zrange([this.toProcessRedisKey(), 0, -1]);
-    if (!onlyKeys) {
-      request = request.hgetall(this.queriesDefKey());
-    }
-    const [active, toProcess, allQueryDefs] = await request.execAsync();
-    return [active, toProcess, R.map(q => JSON.parse(q), allQueryDefs || {})];
-  }
-
-  async getQueryDef(queryKey) {
-    const query = await this.redisClient.hgetAsync([this.queriesDefKey(), this.redisHash(queryKey)]);
-    return JSON.parse(query);
-  }
-
-  updateHeartBeat(queryKey) {
-    return this.redisClient.zaddAsync([this.heartBeatRedisKey(), new Date().getTime(), this.redisHash(queryKey)]);
-  }
-
-  async getNextProcessingId() {
-    const id = await this.redisClient.incrAsync(this.processingIdKey());
-    return id && id.toString();
-  }
-
-  async retrieveForProcessing(queryKey, processingId) {
-    try {
-      const lockKey = this.queryProcessingLockKey(queryKey);
-      await this.redisClient.watchAsync(lockKey);
-
-      const currentProcessId = await this.redisClient.getAsync(lockKey);
-
-      if (currentProcessId) {
-        return null;
-      }
-
-      const result =
-        await this.redisClient.multi()
-          .zadd([this.activeRedisKey(), 'NX', processingId, this.redisHash(queryKey)])
-          .zremrangebyrank([this.activeRedisKey(), this.concurrency, -1])
-          .zrange([this.activeRedisKey(), 0, this.concurrency - 1])
-          .zcard(this.toProcessRedisKey())
-          .hget(([this.queriesDefKey(), this.redisHash(queryKey)]))
-          .set(lockKey, processingId, 'NX')
-          .zadd([this.heartBeatRedisKey(), 'NX', new Date().getTime(), this.redisHash(queryKey)])
-          .execAsync();
-      if (result) {
-        result[4] = JSON.parse(result[4]);
-      }
-      return result;
-    } finally {
-      await this.redisClient.unwatchAsync();
-    }
-  }
-
-  async freeProcessingLock(queryKey, processingId, activated) {
-    try {
-      const lockKey = this.queryProcessingLockKey(queryKey);
-      await this.redisClient.watchAsync(lockKey);
-      const currentProcessId = await this.redisClient.getAsync(lockKey);
-      if (currentProcessId === processingId) {
-        let removeCommand = this.redisClient.multi()
-          .del(lockKey);
-        if (activated) {
-          removeCommand = removeCommand.zrem([this.activeRedisKey(), this.redisHash(queryKey)]);
-        }
-        await removeCommand
-          .execAsync();
-        return null;
-      } else {
-        return currentProcessId;
-      }
-    } finally {
-      await this.redisClient.unwatchAsync();
-    }
-  }
-
-  async optimisticQueryUpdate(queryKey, toUpdate, processingId) {
-    try {
-      let query = await this.getQueryDef(queryKey);
-      for (let i = 0; i < 10; i++) {
-        if (query) {
-          // eslint-disable-next-line no-await-in-loop
-          await this.redisClient.watchAsync(this.queryProcessingLockKey(queryKey));
-          const currentProcessId = await this.redisClient.getAsync(this.queryProcessingLockKey(queryKey));
-          if (currentProcessId !== processingId) {
-            return false;
-          }
-          let [beforeUpdate] = await this.redisClient
-            .multi()
-            .hget([this.queriesDefKey(), this.redisHash(queryKey)])
-            .hset([this.queriesDefKey(), this.redisHash(queryKey), JSON.stringify({ ...query, ...toUpdate })])
-            .execAsync();
-          beforeUpdate = JSON.parse(beforeUpdate);
-          if (JSON.stringify(query) === JSON.stringify(beforeUpdate)) {
-            return true;
-          }
-          query = beforeUpdate;
-        }
-      }
-      throw new Error(`Can't update ${queryKey} with ${JSON.stringify(toUpdate)}`);
-    } finally {
-      await this.redisClient.unwatchAsync();
-    }
-  }
-
-  release() {
-    return this.redisClient.quit();
-  }
-
-  toProcessRedisKey() {
-    return this.queueRedisKey('QUEUE');
-  }
-
-  recentRedisKey() {
-    return this.queueRedisKey('RECENT');
-  }
-
-  activeRedisKey() {
-    return this.queueRedisKey('ACTIVE');
-  }
-
-  heartBeatRedisKey() {
-    return this.queueRedisKey('HEART_BEAT');
-  }
-
-  queryRedisKey(queryKey, suffix) {
-    return `${this.redisQueuePrefix}_${this.redisHash(queryKey)}_${suffix}`;
-  }
-
-  queueRedisKey(suffix) {
-    return `${this.redisQueuePrefix}_${suffix}`;
-  }
-
-  queriesDefKey() {
-    return this.queueRedisKey('QUERIES');
-  }
-
-  processingIdKey() {
-    return this.queueRedisKey('PROCESSING_COUNTER');
-  }
-
-  resultListKey(queryKey) {
-    return this.queryRedisKey(queryKey, 'RESULT');
-  }
-
-  queryProcessingLockKey(queryKey) {
-    return this.queryRedisKey(queryKey, 'LOCK');
-  }
-
-  redisHash(queryKey) {
-    return this.driver.redisHash(queryKey);
-  }
-}
-
-class RedisQueueDriver extends BaseQueueDriver {
-  constructor(options) {
-    super();
-    this.redisPool = options.redisPool;
-    this.options = options;
-  }
-
-  async createConnection() {
-    const redisClient = await this.redisPool.getClient();
-    return new RedisQueueDriverConnection(this, {
-      redisClient,
-      ...this.options
-    });
-  }
-
-  release(connection) {
-    this.redisPool.release(connection.redisClient);
-  }
-}
+process.emitWarning(
+  'Using absolute import with @cubejs-backend/query-orchestrator is deprecated',
+  'DeprecationWarning'
+);
 
 module.exports = RedisQueueDriver;

package/orchestrator/TimeoutError.js

@@ -1,3 +1,8 @@
-class TimeoutError extends Error {}
+const { TimeoutError } = require('../dist/src/orchestrator/TimeoutError');
+
+process.emitWarning(
+  'Using absolute import with @cubejs-backend/query-orchestrator is deprecated',
+  'DeprecationWarning'
+);
 
 module.exports = TimeoutError;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@cubejs-backend/query-orchestrator",
   "description": "Cube.js Query Orchestrator and Cache",
   "author": "Cube Dev, Inc.",
-  "version": "0.24.0",
+  "version": "0.24.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/cube-js/cube.js.git",
@@ -11,25 +11,43 @@
   "engines": {
     "node": ">=8.11.1"
   },
+  "main": "dist/src/index.js",
+  "typings": "dist/src/index.d.ts",
   "scripts": {
+    "build": "rm -rf dist && npm run tsc",
+    "tsc": "tsc",
+    "watch": "tsc -w",
     "test": "npm run unit && npm run integration",
-    "unit": "jest --runInBand --verbose test/unit",
-    "integration": "jest --runInBand --verbose test/integration",
-    "integration:redis": "jest --runInBand --verbose test/integration",
-    "lint": "eslint orchestrator/*.js driver/*.js test/**/*.js"
+    "unit": "jest --runInBand --verbose dist/test/unit",
+    "integration": "npm run integration:redis",
+    "integration:redis": "jest --runInBand --verbose dist/test/integration",
+    "lint": "./node_modules/@cubejs-backend/linter/node_modules/.bin/eslint src/* test/* --ext .ts,.js",
+    "lint:fix": "./node_modules/@cubejs-backend/linter/node_modules/.bin/eslint --fix src/* test/* --ext .ts,.js"
   },
+  "files": [
+    "README.md",
+    "driver/*",
+    "orchestrator/*",
+    "dist/src/*"
+  ],
   "dependencies": {
     "generic-pool": "^3.7.1",
     "ramda": "^0.27.0",
     "redis": "^3.0.2"
   },
   "devDependencies": {
-    "eslint": "^6.8.0",
-    "eslint-config-airbnb-base": "^13.1.0",
-    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-node": "^5.2.1",
-    "jest": "^25.1.0"
+    "@cubejs-backend/linter": "^0.24.2",
+    "@types/generic-pool": "^3.1.9",
+    "@types/jest": "^26.0.15",
+    "@types/node": "^8.10.66",
+    "@types/ramda": "^0.27.32",
+    "@types/redis": "^2.8.28",
+    "jest": "^26.6.3",
+    "typescript": "^4.0.5"
   },
   "license": "Apache-2.0",
-  "gitHead": "04d0fe148181b6e19cdfe53ef9f9cebc3b3ba766"
+  "eslintConfig": {
+    "extends": "../cubejs-linter"
+  },
+  "gitHead": "20a2c50c07d1b7602c0958fd3d5a69f310a415f1"
 }

package/docker-compose.yml

@@ -1,7 +0,0 @@
-version: "3.2"
-services:
-
-  redis:
-    image: "redis:alpine"
-    ports:
-      - "6379:6379"

package/test/integration/QueryQueueRedis.test.js

@@ -1,5 +0,0 @@
-const QueryQueueTest = require('../unit/QueryQueue.test');
-const RedisPool = require('../../orchestrator/RedisPool');
-
-QueryQueueTest('RedisPool', { cacheAndQueueDriver: 'redis', redisPool: new RedisPool() });
-QueryQueueTest('RedisNoPool', { cacheAndQueueDriver: 'redis', redisPool: new RedisPool({ poolMin: 0, poolMax: 0 }) });