@cubejs-backend/query-orchestrator 0.23.3 → 0.24.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/CHANGELOG.md +38 -0
  2. package/README.md +2 -2
  3. package/dist/src/driver/BaseDriver.d.ts +36 -0
  4. package/dist/src/driver/BaseDriver.d.ts.map +1 -0
  5. package/dist/src/driver/BaseDriver.js +175 -0
  6. package/dist/src/driver/BaseDriver.js.map +1 -0
  7. package/dist/src/driver/index.d.ts +3 -0
  8. package/dist/src/driver/index.d.ts.map +1 -0
  9. package/dist/src/driver/index.js +15 -0
  10. package/dist/src/driver/index.js.map +1 -0
  11. package/dist/src/driver/utils.d.ts +2 -0
  12. package/dist/src/driver/utils.d.ts.map +1 -0
  13. package/dist/src/driver/utils.js +17 -0
  14. package/dist/src/driver/utils.js.map +1 -0
  15. package/dist/src/index.d.ts +3 -0
  16. package/dist/src/index.d.ts.map +1 -0
  17. package/dist/src/index.js +15 -0
  18. package/dist/src/index.js.map +1 -0
  19. package/dist/src/orchestrator/BaseQueueDriver.d.ts +4 -0
  20. package/dist/src/orchestrator/BaseQueueDriver.d.ts.map +1 -0
  21. package/dist/src/orchestrator/BaseQueueDriver.js +16 -0
  22. package/dist/src/orchestrator/BaseQueueDriver.js.map +1 -0
  23. package/dist/src/orchestrator/ContinueWaitError.d.ts +3 -0
  24. package/dist/src/orchestrator/ContinueWaitError.d.ts.map +1 -0
  25. package/dist/src/orchestrator/ContinueWaitError.js +10 -0
  26. package/dist/src/orchestrator/ContinueWaitError.js.map +1 -0
  27. package/dist/src/orchestrator/LocalCacheDriver.d.ts +8 -0
  28. package/dist/src/orchestrator/LocalCacheDriver.d.ts.map +1 -0
  29. package/dist/src/orchestrator/LocalCacheDriver.js +30 -0
  30. package/dist/src/orchestrator/LocalCacheDriver.js.map +1 -0
  31. package/dist/src/orchestrator/LocalQueueDriver.d.ts +57 -0
  32. package/dist/src/orchestrator/LocalQueueDriver.d.ts.map +1 -0
  33. package/dist/src/orchestrator/LocalQueueDriver.js +230 -0
  34. package/dist/src/orchestrator/LocalQueueDriver.js.map +1 -0
  35. package/dist/src/orchestrator/PreAggregations.d.ts +26 -0
  36. package/dist/src/orchestrator/PreAggregations.d.ts.map +1 -0
  37. package/dist/src/orchestrator/PreAggregations.js +565 -0
  38. package/dist/src/orchestrator/PreAggregations.js.map +1 -0
  39. package/dist/src/orchestrator/QueryCache.d.ts +51 -0
  40. package/dist/src/orchestrator/QueryCache.d.ts.map +1 -0
  41. package/dist/src/orchestrator/QueryCache.js +293 -0
  42. package/dist/src/orchestrator/QueryCache.js.map +1 -0
  43. package/dist/src/orchestrator/QueryOrchestrator.d.ts +27 -0
  44. package/dist/src/orchestrator/QueryOrchestrator.d.ts.map +1 -0
  45. package/dist/src/orchestrator/QueryOrchestrator.js +79 -0
  46. package/dist/src/orchestrator/QueryOrchestrator.js.map +1 -0
  47. package/dist/src/orchestrator/QueryQueue.d.ts +36 -0
  48. package/dist/src/orchestrator/QueryQueue.d.ts.map +1 -0
  49. package/dist/src/orchestrator/QueryQueue.js +351 -0
  50. package/dist/src/orchestrator/QueryQueue.js.map +1 -0
  51. package/dist/src/orchestrator/RedisCacheDriver.d.ts +12 -0
  52. package/dist/src/orchestrator/RedisCacheDriver.d.ts.map +1 -0
  53. package/dist/src/orchestrator/RedisCacheDriver.js +50 -0
  54. package/dist/src/orchestrator/RedisCacheDriver.js.map +1 -0
  55. package/dist/src/orchestrator/RedisFactory.d.ts +3 -0
  56. package/dist/src/orchestrator/RedisFactory.d.ts.map +1 -0
  57. package/dist/src/orchestrator/RedisFactory.js +45 -0
  58. package/dist/src/orchestrator/RedisFactory.js.map +1 -0
  59. package/dist/src/orchestrator/RedisPool.d.ts +10 -0
  60. package/dist/src/orchestrator/RedisPool.d.ts.map +1 -0
  61. package/dist/src/orchestrator/RedisPool.js +57 -0
  62. package/dist/src/orchestrator/RedisPool.js.map +1 -0
  63. package/dist/src/orchestrator/RedisQueueDriver.d.ts +47 -0
  64. package/dist/src/orchestrator/RedisQueueDriver.d.ts.map +1 -0
  65. package/dist/src/orchestrator/RedisQueueDriver.js +253 -0
  66. package/dist/src/orchestrator/RedisQueueDriver.js.map +1 -0
  67. package/dist/src/orchestrator/TimeoutError.d.ts +4 -0
  68. package/dist/src/orchestrator/TimeoutError.d.ts.map +1 -0
  69. package/dist/src/orchestrator/TimeoutError.js +7 -0
  70. package/dist/src/orchestrator/TimeoutError.js.map +1 -0
  71. package/dist/src/orchestrator/index.d.ts +14 -0
  72. package/dist/src/orchestrator/index.d.ts.map +1 -0
  73. package/dist/src/orchestrator/index.js +26 -0
  74. package/dist/src/orchestrator/index.js.map +1 -0
  75. package/driver/BaseDriver.js +5 -221
  76. package/driver/README.md +3 -0
  77. package/driver/utils.js +8 -12
  78. package/orchestrator/BaseQueueDriver.js +5 -8
  79. package/orchestrator/ContinueWaitError.js +6 -5
  80. package/orchestrator/LocalCacheDriver.js +5 -29
  81. package/orchestrator/LocalQueueDriver.js +5 -256
  82. package/orchestrator/PreAggregations.js +4 -764
  83. package/orchestrator/QueryCache.js +5 -381
  84. package/orchestrator/QueryOrchestrator.js +5 -100
  85. package/orchestrator/QueryQueue.js +5 -378
  86. package/orchestrator/README.md +3 -0
  87. package/orchestrator/RedisCacheDriver.js +5 -45
  88. package/orchestrator/RedisFactory.js +6 -46
  89. package/orchestrator/RedisPool.js +5 -49
  90. package/orchestrator/RedisQueueDriver.js +5 -283
  91. package/orchestrator/TimeoutError.js +6 -1
  92. package/package.json +29 -10
  93. package/test/integration/PreAggregations.test.js +0 -337
  94. package/test/integration/QueryQueueRedis.test.js +0 -6
  95. package/test/unit/QueryOrchestrator.test.js +0 -301
  96. package/test/unit/QueryQueue.test.js +0 -249
@@ -1,286 +1,8 @@
1
- const R = require('ramda');
1
+ const { RedisQueueDriver } = require('../dist/src/orchestrator/RedisQueueDriver');
2
2
 
3
- const BaseQueueDriver = require('./BaseQueueDriver');
4
-
5
- class RedisQueueDriverConnection {
6
- constructor(driver, options) {
7
- this.driver = driver;
8
- this.redisClient = options.redisClient;
9
- this.redisQueuePrefix = options.redisQueuePrefix;
10
- this.continueWaitTimeout = options.continueWaitTimeout;
11
- this.orphanedTimeout = options.orphanedTimeout;
12
- this.heartBeatTimeout = options.heartBeatTimeout;
13
- this.concurrency = options.concurrency;
14
- }
15
-
16
- async getResultBlocking(queryKey) {
17
- const resultListKey = this.resultListKey(queryKey);
18
- if (!(await this.redisClient.hgetAsync([this.queriesDefKey(), this.redisHash(queryKey)]))) {
19
- return this.getResult(queryKey);
20
- }
21
- const result = await this.redisClient.brpopAsync([resultListKey, this.continueWaitTimeout]);
22
- if (result) {
23
- await this.redisClient.lpushAsync([resultListKey, result[1]]);
24
- await this.redisClient.rpopAsync(resultListKey);
25
- }
26
- return result && JSON.parse(result[1]);
27
- }
28
-
29
- async getResult(queryKey) {
30
- const resultListKey = this.resultListKey(queryKey);
31
- const result = await this.redisClient.rpopAsync(resultListKey);
32
- return result && JSON.parse(result);
33
- }
34
-
35
- addToQueue(keyScore, queryKey, time, queryHandler, query, priority, options) {
36
- return this.redisClient.multi()
37
- .zadd([this.toProcessRedisKey(), 'NX', keyScore, this.redisHash(queryKey)])
38
- .zadd([this.recentRedisKey(), time, this.redisHash(queryKey)])
39
- .hsetnx([
40
- this.queriesDefKey(),
41
- this.redisHash(queryKey),
42
- JSON.stringify({
43
- queryHandler,
44
- query,
45
- queryKey,
46
- stageQueryKey: options.stageQueryKey,
47
- priority,
48
- requestId: options.requestId,
49
- addedToQueueTime: new Date().getTime()
50
- })
51
- ])
52
- .zcard(this.toProcessRedisKey())
53
- .execAsync();
54
- }
55
-
56
- getToProcessQueries() {
57
- return this.redisClient.zrangeAsync([this.toProcessRedisKey(), 0, -1]);
58
- }
59
-
60
- getActiveQueries() {
61
- return this.redisClient.zrangeAsync([this.activeRedisKey(), 0, -1]);
62
- }
63
-
64
- async getQueryAndRemove(queryKey) {
65
- const [query, ...restResult] = await this.redisClient.multi()
66
- .hget([this.queriesDefKey(), this.redisHash(queryKey)])
67
- .zrem([this.activeRedisKey(), this.redisHash(queryKey)])
68
- .zrem([this.heartBeatRedisKey(), this.redisHash(queryKey)])
69
- .zrem([this.toProcessRedisKey(), this.redisHash(queryKey)])
70
- .zrem([this.recentRedisKey(), this.redisHash(queryKey)])
71
- .hdel([this.queriesDefKey(), this.redisHash(queryKey)])
72
- .del(this.queryProcessingLockKey(queryKey))
73
- .execAsync();
74
- return [JSON.parse(query), ...restResult];
75
- }
76
-
77
- async setResultAndRemoveQuery(queryKey, executionResult, processingId) {
78
- try {
79
- await this.redisClient.watchAsync(this.queryProcessingLockKey(queryKey));
80
- const currentProcessId = await this.redisClient.getAsync(this.queryProcessingLockKey(queryKey));
81
- if (processingId !== currentProcessId) {
82
- return false;
83
- }
84
-
85
- return this.redisClient.multi()
86
- .lpush([this.resultListKey(queryKey), JSON.stringify(executionResult)])
87
- .zrem([this.activeRedisKey(), this.redisHash(queryKey)])
88
- .zrem([this.heartBeatRedisKey(), this.redisHash(queryKey)])
89
- .zrem([this.toProcessRedisKey(), this.redisHash(queryKey)])
90
- .zrem([this.recentRedisKey(), this.redisHash(queryKey)])
91
- .hdel([this.queriesDefKey(), this.redisHash(queryKey)])
92
- .del(this.queryProcessingLockKey(queryKey))
93
- .execAsync();
94
- } finally {
95
- await this.redisClient.unwatchAsync();
96
- }
97
- }
98
-
99
- getOrphanedQueries() {
100
- return this.redisClient.zrangebyscoreAsync(
101
- [this.recentRedisKey(), 0, (new Date().getTime() - this.orphanedTimeout * 1000)]
102
- );
103
- }
104
-
105
- getStalledQueries() {
106
- return this.redisClient.zrangebyscoreAsync(
107
- [this.heartBeatRedisKey(), 0, (new Date().getTime() - this.heartBeatTimeout * 1000)]
108
- );
109
- }
110
-
111
- async getQueryStageState(onlyKeys) {
112
- let request = this.redisClient.multi()
113
- .zrange([this.activeRedisKey(), 0, -1])
114
- .zrange([this.toProcessRedisKey(), 0, -1]);
115
- if (!onlyKeys) {
116
- request = request.hgetall(this.queriesDefKey());
117
- }
118
- const [active, toProcess, allQueryDefs] = await request.execAsync();
119
- return [active, toProcess, R.map(q => JSON.parse(q), allQueryDefs || {})];
120
- }
121
-
122
- async getQueryDef(queryKey) {
123
- const query = await this.redisClient.hgetAsync([this.queriesDefKey(), this.redisHash(queryKey)]);
124
- return JSON.parse(query);
125
- }
126
-
127
- updateHeartBeat(queryKey) {
128
- return this.redisClient.zaddAsync([this.heartBeatRedisKey(), new Date().getTime(), this.redisHash(queryKey)]);
129
- }
130
-
131
- async getNextProcessingId() {
132
- const id = await this.redisClient.incrAsync(this.processingIdKey());
133
- return id && id.toString();
134
- }
135
-
136
- async retrieveForProcessing(queryKey, processingId) {
137
- try {
138
- const lockKey = this.queryProcessingLockKey(queryKey);
139
- await this.redisClient.watchAsync(lockKey);
140
-
141
- const currentProcessId = await this.redisClient.getAsync(lockKey);
142
-
143
- if (currentProcessId) {
144
- return null;
145
- }
146
-
147
- const result =
148
- await this.redisClient.multi()
149
- .zadd([this.activeRedisKey(), 'NX', processingId, this.redisHash(queryKey)])
150
- .zremrangebyrank([this.activeRedisKey(), this.concurrency, -1])
151
- .zrange([this.activeRedisKey(), 0, this.concurrency - 1])
152
- .zcard(this.toProcessRedisKey())
153
- .hget(([this.queriesDefKey(), this.redisHash(queryKey)]))
154
- .set(lockKey, processingId, 'NX')
155
- .zadd([this.heartBeatRedisKey(), 'NX', new Date().getTime(), this.redisHash(queryKey)])
156
- .execAsync();
157
- if (result) {
158
- result[4] = JSON.parse(result[4]);
159
- }
160
- return result;
161
- } finally {
162
- await this.redisClient.unwatchAsync();
163
- }
164
- }
165
-
166
- async freeProcessingLock(queryKey, processingId, activated) {
167
- try {
168
- const lockKey = this.queryProcessingLockKey(queryKey);
169
- await this.redisClient.watchAsync(lockKey);
170
- const currentProcessId = await this.redisClient.getAsync(lockKey);
171
- if (currentProcessId === processingId) {
172
- let removeCommand = this.redisClient.multi()
173
- .del(lockKey);
174
- if (activated) {
175
- removeCommand = removeCommand.zrem([this.activeRedisKey(), this.redisHash(queryKey)]);
176
- }
177
- await removeCommand
178
- .execAsync();
179
- return null;
180
- } else {
181
- return currentProcessId;
182
- }
183
- } finally {
184
- await this.redisClient.unwatchAsync();
185
- }
186
- }
187
-
188
- async optimisticQueryUpdate(queryKey, toUpdate, processingId) {
189
- try {
190
- let query = await this.getQueryDef(queryKey);
191
- for (let i = 0; i < 10; i++) {
192
- if (query) {
193
- // eslint-disable-next-line no-await-in-loop
194
- await this.redisClient.watchAsync(this.queryProcessingLockKey(queryKey));
195
- const currentProcessId = await this.redisClient.getAsync(this.queryProcessingLockKey(queryKey));
196
- if (currentProcessId !== processingId) {
197
- return false;
198
- }
199
- let [beforeUpdate] = await this.redisClient
200
- .multi()
201
- .hget([this.queriesDefKey(), this.redisHash(queryKey)])
202
- .hset([this.queriesDefKey(), this.redisHash(queryKey), JSON.stringify({ ...query, ...toUpdate })])
203
- .execAsync();
204
- beforeUpdate = JSON.parse(beforeUpdate);
205
- if (JSON.stringify(query) === JSON.stringify(beforeUpdate)) {
206
- return true;
207
- }
208
- query = beforeUpdate;
209
- }
210
- }
211
- throw new Error(`Can't update ${queryKey} with ${JSON.stringify(toUpdate)}`);
212
- } finally {
213
- await this.redisClient.unwatchAsync();
214
- }
215
- }
216
-
217
- release() {
218
- return this.redisClient.quit();
219
- }
220
-
221
- toProcessRedisKey() {
222
- return this.queueRedisKey('QUEUE');
223
- }
224
-
225
- recentRedisKey() {
226
- return this.queueRedisKey('RECENT');
227
- }
228
-
229
- activeRedisKey() {
230
- return this.queueRedisKey('ACTIVE');
231
- }
232
-
233
- heartBeatRedisKey() {
234
- return this.queueRedisKey('HEART_BEAT');
235
- }
236
-
237
- queryRedisKey(queryKey, suffix) {
238
- return `${this.redisQueuePrefix}_${this.redisHash(queryKey)}_${suffix}`;
239
- }
240
-
241
- queueRedisKey(suffix) {
242
- return `${this.redisQueuePrefix}_${suffix}`;
243
- }
244
-
245
- queriesDefKey() {
246
- return this.queueRedisKey('QUERIES');
247
- }
248
-
249
- processingIdKey() {
250
- return this.queueRedisKey('PROCESSING_COUNTER');
251
- }
252
-
253
- resultListKey(queryKey) {
254
- return this.queryRedisKey(queryKey, 'RESULT');
255
- }
256
-
257
- queryProcessingLockKey(queryKey) {
258
- return this.queryRedisKey(queryKey, 'LOCK');
259
- }
260
-
261
- redisHash(queryKey) {
262
- return this.driver.redisHash(queryKey);
263
- }
264
- }
265
-
266
- class RedisQueueDriver extends BaseQueueDriver {
267
- constructor(options) {
268
- super();
269
- this.redisPool = options.redisPool;
270
- this.options = options;
271
- }
272
-
273
- async createConnection() {
274
- const redisClient = await this.redisPool.getClient();
275
- return new RedisQueueDriverConnection(this, {
276
- redisClient,
277
- ...this.options
278
- });
279
- }
280
-
281
- release(connection) {
282
- this.redisPool.release(connection.redisClient);
283
- }
284
- }
3
+ process.emitWarning(
4
+ 'Using absolute import with @cubejs-backend/query-orchestrator is deprecated',
5
+ 'DeprecationWarning'
6
+ );
285
7
 
286
8
  module.exports = RedisQueueDriver;
@@ -1,3 +1,8 @@
1
- class TimeoutError extends Error {}
1
+ const { TimeoutError } = require('../dist/src/orchestrator/TimeoutError');
2
+
3
+ process.emitWarning(
4
+ 'Using absolute import with @cubejs-backend/query-orchestrator is deprecated',
5
+ 'DeprecationWarning'
6
+ );
2
7
 
3
8
  module.exports = TimeoutError;
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@cubejs-backend/query-orchestrator",
3
3
  "description": "Cube.js Query Orchestrator and Cache",
4
4
  "author": "Cube Dev, Inc.",
5
- "version": "0.23.3",
5
+ "version": "0.24.2",
6
6
  "repository": {
7
7
  "type": "git",
8
8
  "url": "https://github.com/cube-js/cube.js.git",
@@ -11,24 +11,43 @@
11
11
  "engines": {
12
12
  "node": ">=8.11.1"
13
13
  },
14
+ "main": "dist/src/index.js",
15
+ "typings": "dist/src/index.d.ts",
14
16
  "scripts": {
17
+ "build": "rm -rf dist && npm run tsc",
18
+ "tsc": "tsc",
19
+ "watch": "tsc -w",
15
20
  "test": "npm run unit && npm run integration",
16
- "unit": "jest --runInBand --verbose test/unit",
17
- "integration": "jest --runInBand --verbose test/integration",
18
- "lint": "eslint orchestrator/*.js driver/*.js"
21
+ "unit": "jest --runInBand --verbose dist/test/unit",
22
+ "integration": "npm run integration:redis",
23
+ "integration:redis": "jest --runInBand --verbose dist/test/integration",
24
+ "lint": "./node_modules/@cubejs-backend/linter/node_modules/.bin/eslint src/* test/* --ext .ts,.js",
25
+ "lint:fix": "./node_modules/@cubejs-backend/linter/node_modules/.bin/eslint --fix src/* test/* --ext .ts,.js"
19
26
  },
27
+ "files": [
28
+ "README.md",
29
+ "driver/*",
30
+ "orchestrator/*",
31
+ "dist/src/*"
32
+ ],
20
33
  "dependencies": {
21
34
  "generic-pool": "^3.7.1",
22
35
  "ramda": "^0.27.0",
23
36
  "redis": "^3.0.2"
24
37
  },
25
38
  "devDependencies": {
26
- "eslint": "^6.8.0",
27
- "eslint-config-airbnb-base": "^13.1.0",
28
- "eslint-plugin-import": "^2.16.0",
29
- "eslint-plugin-node": "^5.2.1",
30
- "jest": "^25.1.0"
39
+ "@cubejs-backend/linter": "^0.24.2",
40
+ "@types/generic-pool": "^3.1.9",
41
+ "@types/jest": "^26.0.15",
42
+ "@types/node": "^8.10.66",
43
+ "@types/ramda": "^0.27.32",
44
+ "@types/redis": "^2.8.28",
45
+ "jest": "^26.6.3",
46
+ "typescript": "^4.0.5"
31
47
  },
32
48
  "license": "Apache-2.0",
33
- "gitHead": "ae0115d4f65ddc4f91c38fa8325dc2252f892706"
49
+ "eslintConfig": {
50
+ "extends": "../cubejs-linter"
51
+ },
52
+ "gitHead": "20a2c50c07d1b7602c0958fd3d5a69f310a415f1"
34
53
  }
@@ -1,337 +0,0 @@
1
- /* globals describe, beforeAll, afterAll, beforeEach, test, expect */
2
- const R = require('ramda');
3
-
4
- class MockDriver {
5
- constructor() {
6
- this.tables = [];
7
- this.executedQueries = [];
8
- this.cancelledQueries = [];
9
- }
10
-
11
- query(query) {
12
- this.executedQueries.push(query);
13
- let promise = Promise.resolve([query]);
14
- if (query.match(`orders_too_big`)) {
15
- promise = promise.then((res) => new Promise(resolve => setTimeout(() => resolve(res), 3000)));
16
- }
17
- promise.cancel = async () => {
18
- this.cancelledQueries.push(query);
19
- };
20
- return promise;
21
- }
22
-
23
- async getTablesQuery(schema) {
24
- return this.tables.map(t => ({ table_name: t.replace(`${schema}.`, '') }));
25
- }
26
-
27
- async createSchemaIfNotExists(schema) {
28
- this.schema = schema;
29
- return null;
30
- }
31
-
32
- loadPreAggregationIntoTable(preAggregationTableName, loadSql) {
33
- this.tables.push(preAggregationTableName.substring(0, 100));
34
- return this.query(loadSql);
35
- }
36
-
37
- async dropTable(tableName) {
38
- this.tables = this.tables.filter(t => t !== tableName);
39
- return this.query(`DROP TABLE ${tableName}`);
40
- }
41
-
42
- async downloadTable(table) {
43
- return { rows: await this.query(`SELECT * FROM ${table}`) };
44
- }
45
-
46
- async tableColumnTypes(table) {
47
- return [];
48
- }
49
-
50
- async uploadTable(table, columns, tableData) {
51
- await this.createTable(table, columns);
52
- }
53
-
54
- createTable(quotedTableName, columns) {
55
- this.tables.push(quotedTableName);
56
- }
57
-
58
- readOnly() {
59
- return false;
60
- }
61
- }
62
-
63
- describe('PreAggregations', () => {
64
- let mockDriver = null;
65
- let mockExternalDriver = null;
66
- let mockDriverFactory = null;
67
- let mockDriverReadOnlyFactory = null;
68
- let mockExternalDriverFactory = null;
69
- let queryCache = null;
70
- const basicQuery = {
71
- query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count20191101) as partition_union WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
72
- values: ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"],
73
- cacheKeyQueries: {
74
- renewalThreshold: 21600,
75
- queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
76
- },
77
- preAggregations: [{
78
- preAggregationsSchema: "stb_pre_aggregations",
79
- tableName: "stb_pre_aggregations.orders_number_and_count20191101",
80
- loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count20191101 AS SELECT\n date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n FROM\n public.orders AS \"orders\"\n WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"]],
81
- invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
82
- }],
83
- requestId: 'basic'
84
- };
85
- const basicQueryExternal = R.clone(basicQuery);
86
- basicQueryExternal.preAggregations[0].external = true;
87
- const basicQueryWithRenew = R.clone(basicQuery);
88
- basicQueryWithRenew.renewQuery = true;
89
- const basicQueryExternalWithRenew = R.clone(basicQueryExternal);
90
- basicQueryExternalWithRenew.renewQuery = true;
91
-
92
- beforeEach(() => {
93
- mockDriver = new MockDriver();
94
- mockExternalDriver = new MockDriver();
95
- mockDriverFactory = async () => mockDriver;
96
- mockDriverReadOnlyFactory = async () => {
97
- const driver = mockDriver;
98
- jest.spyOn(driver, 'readOnly').mockImplementation(() => true);
99
- return driver;
100
- }
101
- mockExternalDriverFactory = async () => {
102
- const driver = mockExternalDriver;
103
- driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209");
104
- return driver;
105
- }
106
-
107
- jest.resetModules();
108
- const QueryCache = require('../../orchestrator/QueryCache');
109
- queryCache = new QueryCache(
110
- "TEST",
111
- mockDriverFactory,
112
- (msg, params) => {},
113
- {
114
- queueOptions: {
115
- executionTimeout: 1
116
- },
117
- },
118
- );
119
- });
120
-
121
- describe('loadAllPreAggregationsIfNeeded', () => {
122
- let preAggregations = null;
123
-
124
- beforeEach(async () => {
125
- const PreAggregations = require('../../orchestrator/PreAggregations');
126
- preAggregations = new PreAggregations(
127
- "TEST",
128
- mockDriverFactory,
129
- (msg, params) => {},
130
- queryCache,
131
- {
132
- queueOptions: {
133
- executionTimeout: 1
134
- },
135
- },
136
- );
137
- });
138
-
139
- test('syncronously create rollup from scratch', async () => {
140
- const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryWithRenew);
141
- expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/);
142
- });
143
- });
144
-
145
- describe(`loadAllPreAggregationsIfNeeded with external rollup and writable source`, () => {
146
- let preAggregations = null;
147
-
148
- beforeEach(async () => {
149
- const PreAggregations = require('../../orchestrator/PreAggregations');
150
- preAggregations = new PreAggregations(
151
- "TEST",
152
- mockDriverFactory,
153
- (msg, params) => {},
154
- queryCache,
155
- {
156
- queueOptions: {
157
- executionTimeout: 1
158
- },
159
- externalDriverFactory: mockExternalDriverFactory,
160
- },
161
- );
162
- });
163
-
164
- test('refresh external preaggregation with a writable source (refreshImplTempTableExternalStrategy)', async () => {
165
- const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
166
- expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/);
167
- });
168
- });
169
-
170
- describe(`loadAllPreAggregationsIfNeeded with external rollup and readonly source`, () => {
171
- let preAggregations = null;
172
-
173
- beforeEach(async () => {
174
- const PreAggregations = require('../../orchestrator/PreAggregations');
175
- preAggregations = new PreAggregations(
176
- "TEST",
177
- mockDriverReadOnlyFactory,
178
- (msg, params) => {},
179
- queryCache,
180
- {
181
- queueOptions: {
182
- executionTimeout: 1
183
- },
184
- externalDriverFactory: mockExternalDriverFactory,
185
- },
186
- );
187
- });
188
-
189
- test('refresh external preaggregation with a writable source (refreshImplStreamExternalStrategy)', async () => {
190
- const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
191
- expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/);
192
- });
193
- });
194
-
195
- describe(`loadAllPreAggregationsIfNeeded with externalRefresh true`, () => {
196
- let preAggregations = null;
197
-
198
- beforeEach(async () => {
199
- const PreAggregations = require('../../orchestrator/PreAggregations');
200
- preAggregations = new PreAggregations(
201
- "TEST",
202
- mockDriverFactory,
203
- (msg, params) => {},
204
- queryCache,
205
- {
206
- queueOptions: {
207
- executionTimeout: 1
208
- },
209
- externalRefresh: true,
210
- },
211
- );
212
- });
213
-
214
- test('fail if waitForRenew is also specified', async () => {
215
- await expect(preAggregations.loadAllPreAggregationsIfNeeded(basicQueryWithRenew))
216
- .rejects.toThrowError(/Invalid configuration/);
217
- });
218
-
219
- test('fail if rollup doesn\'t already exist', async () => {
220
- await expect(preAggregations.loadAllPreAggregationsIfNeeded(basicQuery))
221
- .rejects.toThrowError(/One or more pre-aggregation tables could not be found to satisfy that query/);
222
- });
223
- });
224
-
225
- describe(`loadAllPreAggregationsIfNeeded with external rollup and externalRefresh true`, () => {
226
- let preAggregations = null;
227
-
228
- beforeEach(async () => {
229
- const PreAggregations = require('../../orchestrator/PreAggregations');
230
- preAggregations = new PreAggregations(
231
- "TEST",
232
- () => { throw new Error('The source database factory should never be called when externalRefresh is true, as it will trigger testConnection'); },
233
- (msg, params) => {},
234
- queryCache,
235
- {
236
- queueOptions: {
237
- executionTimeout: 1
238
- },
239
- externalDriverFactory: mockExternalDriverFactory,
240
- externalRefresh: true,
241
- },
242
- );
243
- });
244
-
245
- test('fail if waitForRenew is also specified', async () => {
246
- await expect(preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternalWithRenew))
247
- .rejects.toThrowError(/Invalid configuration/);
248
- });
249
-
250
- test('load external preaggregation without communicating to the source database', async () => {
251
- const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
252
- expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/);
253
- });
254
- });
255
-
256
- describe(`naming_version tests`, () => {
257
- let preAggregations = null;
258
- let PreAggregations = null;
259
-
260
- beforeEach(async () => {
261
- PreAggregations = require('../orchestrator/PreAggregations');
262
- preAggregations = new PreAggregations(
263
- "TEST",
264
- mockDriverFactory,
265
- (msg, params) => {},
266
- queryCache,
267
- {
268
- queueOptions: {
269
- executionTimeout: 1
270
- },
271
- externalDriverFactory: async () => {
272
- const driver = mockExternalDriver;
273
- driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209");
274
- driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652");
275
- return driver;
276
- },
277
- },
278
- );
279
- });
280
-
281
- test('test for function targetTableName', () => {
282
- let result = PreAggregations.targetTableName({
283
- table_name:'orders_number_and_count20191101',
284
- content_version:'kjypcoio',
285
- structure_version:'5yftl5il',
286
- last_updated_at:1600329890789,
287
- });
288
- expect(result).toEqual('orders_number_and_count20191101_kjypcoio_5yftl5il_1600329890789')
289
-
290
- result = PreAggregations.targetTableName({
291
- table_name:'orders_number_and_count20191101',
292
- content_version:'kjypcoio',
293
- structure_version:'5yftl5il',
294
- last_updated_at:1600329890789,
295
- naming_version:2
296
- });
297
- expect(result).toEqual('orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652')
298
- });
299
-
300
- test('naming_version and sort by last_updated_at', async () => {
301
- const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
302
- expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652/);
303
- });
304
- });
305
-
306
- describe(`naming_version sort tests`, () => {
307
- let preAggregations = null;
308
- let PreAggregations = null;
309
-
310
- beforeEach(async () => {
311
- PreAggregations = require('../orchestrator/PreAggregations');
312
- preAggregations = new PreAggregations(
313
- "TEST",
314
- mockDriverFactory,
315
- (msg, params) => {},
316
- queryCache,
317
- {
318
- queueOptions: {
319
- executionTimeout: 1
320
- },
321
- externalDriverFactory: async () => {
322
- const driver = mockExternalDriver;
323
- driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209");
324
- driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652");
325
- return driver;
326
- },
327
- },
328
- );
329
- });
330
-
331
- test('naming_version and sort by last_updated_at', async () => {
332
- const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
333
- expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209/);
334
- });
335
- });
336
-
337
- });