@transitive-sdk/clickhouse 0.2.0 → 0.3.1

package/index.js

@@ -1,6 +1,6 @@
 const _ = require('lodash');
 const { createClient } = require('@clickhouse/client');
-const { topicToPath } = require('@transitive-sdk/datacache');
+const { topicToPath, topicMatch } = require('@transitive-sdk/datacache');
 
 // Default TTL in days for mqtt_history table
 const DEFAULT_TTL_DAYS = 30;
@@ -19,11 +19,28 @@ const MULTI_TENANT_SCHEMA = {
   ]
 };
 
+/** Given a Topic path (array), return an array of conditions to use in a
+ * WHERE clause. */
+const path2where = (path) => {
+  const where = [];
+  _.forEach(path, (value, i) => {
+    if (!['+','#'].includes(value[0])) {
+      // it's a constant, filter by it
+      where.push(`((TopicParts[${i + 1}]) = '${value}')`);
+      // Note that ClickHouse/SQL index starting at 1, not 0
+    }
+  });
+  return where;
+};
+
+
 /** Singleton ClickHouse client wrapper with multi-tenant table support */
 class ClickHouse {
 
   _client = null;
-  mqttHistoryTable = null; /// name of the table used for MQTT history, if used
+
+  mqttHistoryTable = null; // name of the table used for MQTT history, if used
+  topics = {}; // list of topics registered for storage, as object for de-duplication
 
   /** Create the client, connecting to Clickhouse */
   init({ url, dbName, user, password } = {}) {
@@ -120,48 +137,26 @@ class ClickHouse {
     });
   }
 
-  /** Update the TTL for the mqtt_history table
-   * @param {number} ttlDays - TTL in days
+  /* Enable history recording. Ensure the mqtt_history table exists with the
+   * correct schema, set dataCache, and subscribe to changes.
+   * @param {object} options = {dataCache, tableName, ttlDays}
    */
-  async updateMqttHistoryTTL(ttlDays) {
-    // console.log(`updating ttl to ${ttlDays}`);
-    await this.client.command({
-      query: `ALTER TABLE ${this.mqttHistoryTable} MODIFY TTL toDateTime(Timestamp) + toIntervalDay(${ttlDays})`,
-      clickhouse_settings: {
-        wait_end_of_query: 1,
-      }
-    });
-  }
+  async enableHistory(options) {
+    const { dataCache, tableName = 'mqtt_history' } = options;
 
-  /** Ensure the mqtt_history table exists with the correct schema
-   * @param {number} ttlDays - TTL in days (default: 30)
-   */
-  async ensureMqttHistoryTable(tableName = 'mqtt_history', ttlDays = DEFAULT_TTL_DAYS) {
     if (this.mqttHistoryTable != tableName) {
       console.warn(`creating or altering mqtt history table ${tableName}`);
     }
 
-    const ttlExpression = `TTL toDateTime(Timestamp) + toIntervalDay(${ttlDays})`;
-
     // Check if table already exists before creating
     const tableExists = await this.client.query({
-      query: `SELECT name, create_table_query	FROM system.tables WHERE name = '${this.mqttHistoryTable}' AND database = currentDatabase()`,
+      query: `SELECT name	FROM system.tables WHERE name = '${this.mqttHistoryTable}' AND database = currentDatabase()`,
       format: 'JSONEachRow'
     });
     const tables = await tableExists.json();
 
-    if (tables.length > 0) {
-      // table already exists, verify TTL
-      const originalCreateQuery = tables[0].create_table_query;
-
-      // Update table if it differs
-      if (!originalCreateQuery.includes(ttlExpression)) {
-        await this.updateMqttHistoryTTL(ttlDays);
-      }
-
-    } else {
+    if (tables.length == 0) {
       // Create the table
-
       const columns = [
         // High-precision event time; Delta + ZSTD is a common combo for time-series
         'Timestamp DateTime64(6) CODEC(Delta, ZSTD(1))',
@@ -170,29 +165,32 @@ class ClickHouse {
         // Org/device fields materialized from TopicParts (always computed, not overridable)
         'OrgId LowCardinality(String) MATERIALIZED TopicParts[1] CODEC(ZSTD(1))',
         'DeviceId LowCardinality(String) MATERIALIZED TopicParts[2] CODEC(ZSTD(1))',
-        'Scope LowCardinality(String) MATERIALIZED TopicParts[3] CODEC(ZSTD(1))',
-        'CapabilityName LowCardinality(String) MATERIALIZED TopicParts[4] CODEC(ZSTD(1))',
-        'CapabilityVersion LowCardinality(String) MATERIALIZED TopicParts[5] CODEC(ZSTD(1))',
+        // 'Scope LowCardinality(String) MATERIALIZED TopicParts[3] CODEC(ZSTD(1))',
+        // 'CapabilityName LowCardinality(String) MATERIALIZED TopicParts[4] CODEC(ZSTD(1))',
+        // 'CapabilityVersion LowCardinality(String) MATERIALIZED TopicParts[5] CODEC(ZSTD(1))',
         // Remaining topic segments stored as an array for less-structured suffixes
-        'SubTopic Array(String) MATERIALIZED arraySlice(TopicParts, 6) CODEC(ZSTD(1))',
+        // 'SubTopic Array(String) MATERIALIZED arraySlice(TopicParts, 6) CODEC(ZSTD(1))',
         // Payload stored as a String, compressed with ZSTD(1). This allows us to
         // store atomic values (still stringified) as opposed to only JSON objects,
         // as the JSON type would require.
         'Payload String CODEC(ZSTD(1))',
         // Bloom filter indexes (shared multi-tenant indexes)
         ...MULTI_TENANT_SCHEMA.indexes,
-        'INDEX idx_scope (Scope) TYPE bloom_filter(0.01) GRANULARITY 1',
-        'INDEX idx_capability (CapabilityName) TYPE bloom_filter(0.01) GRANULARITY 1'
+        // 'INDEX idx_scope (Scope) TYPE bloom_filter(0.01) GRANULARITY 1',
+        // 'INDEX idx_capability (CapabilityName) TYPE bloom_filter(0.01) GRANULARITY 1'
+        'INDEX idx_scope (TopicParts[3]) TYPE bloom_filter(0.01) GRANULARITY 1',
+        'INDEX idx_capability (TopicParts[4]) TYPE bloom_filter(0.01) GRANULARITY 1'
       ];
 
-      const query = `CREATE TABLE IF NOT EXISTS ${tableName} (${columns.join(', ')})
-        ENGINE = MergeTree()
-        PARTITION BY toYYYYMMDD(Timestamp)
-        ORDER BY (OrgId, toUnixTimestamp64Micro(Timestamp), TopicParts)
-        ${ttlExpression}
-        SETTINGS
-        index_granularity = 8192,
-        ttl_only_drop_parts = 1`;
+      const query = [
+          `CREATE TABLE IF NOT EXISTS default.${tableName} (${columns.join(', ')})`,
+          'ENGINE = MergeTree()',
+          'PARTITION BY toYYYYMMDD(Timestamp)',
+          'ORDER BY (OrgId, toUnixTimestamp64Micro(Timestamp), TopicParts)',
+          'SETTINGS',
+          '  index_granularity = 8192,',
+          '  ttl_only_drop_parts = 1'
+        ].join('\n');
       // Note: PRIMARY KEY is not needed because we want it to be the same as
       // ORDER BY, which is what ClickHouse does automatically.
 
@@ -202,45 +200,130 @@ class ClickHouse {
           wait_end_of_query: 1,
         }
       });
+
+      // grant capabilities read-access to their namespace
+      await this.client.command({ query:
+        `CREATE ROW POLICY IF NOT EXISTS default_capabilities
+        ON default.${tableName}
+        USING TopicParts[3] = splitByString('_', currentUser())[2]
+        AND TopicParts[4] = splitByString('_', currentUser())[3] TO ALL`,
+        clickhouse_settings: {
+          wait_end_of_query: 1,
+        }
+      });
+
+      // Subscribe to changes to the data cache. On each change, check whether
+      // it matches any of the registered topics (this avoid duplicate triggers),
+      // then store to ClickHouse with current timestamp.
+      dataCache.subscribe((changes) => {
+        _.forEach(changes, async (value, topic) => {
+
+          const matched =
+            _.some(this.topics, (_true, selector) => topicMatch(selector, topic));
+
+          if (!matched) return;
+
+          const row = {
+            Timestamp: new Date(),
+            TopicParts: topicToPath(topic), // topic as array
+          };
+
+          if (value !== null && value !== undefined) {
+            row.Payload = JSON.stringify(value);
+          } // else: omit
+
+          try {
+            await this.client.insert({
+              table: this.mqttHistoryTable,
+              values: [row],
+              format: 'JSONEachRow'
+            });
+          } catch (error) {
+            console.error('Error inserting MQTT message into ClickHouse:', error.message);
+          }
+        })
+      });
+
+
     }
 
     this.mqttHistoryTable = tableName;
   }
 
-  /** Register an MQTT topic for storage in ClickHouse subscribes to the topic
+  /* Register an MQTT topic for storage in ClickHouse subscribes to the topic
   * and stores incoming messages JSON.stringify'd in a ClickHouse table.
   * Retrieve using `queryMQTTHistory`, or, when quering directly, e.g., from
   * Grafana, use the ClickHouse built-in functionality for parsing JSON, e.g.,
   * after inserting `{ x: 1 }` use
   * `select JSON_VALUE(Payload, '$.x') AS x FROM default.mqtt_history`.
   * NOTE: `ensureMqttHistoryTable` must be called before registering topics
-  * @param {Object} dataCache - DataCache instance to use for subscribing
   * @param {string} topic - MQTT topic to register
   */
-  registerMqttTopicForStorage(dataCache, topic) {
-    if (!this.mqttHistoryTable) {
-      throw new Error('ensureMqttHistoryTable must be called before registerMqttTopicForStorage');
+  async registerMqttTopicForStorage(selector, ttlDays = DEFAULT_TTL_DAYS) {
+    this.topics[selector] = true;
+
+    // ---------------------------------------------------------------
+    // Set/update TTL for this capability and sub-topic
+
+    const path = topicToPath(selector);
+
+    if (path.length < 4) {
+      // underspecified, don't set TTL
+      return;
+    }
+
+    // list of TopicParts indices and selected value to use in WHERE statement
+    // const topicPartSelectors = [
+    //   [2, path[2]],
+    //   [3, path[3]]
+    // ];
+
+    // path.slice(5).forEach((value, i) => topicPartSelectors.push([i + 5, value]));
+
+    // const where = topicPartSelectors
+    //     // filter out wildcards
+    //     .filter(([i, value]) => !['+','#'].includes(value[0]))
+    //     // map to WHERE conditions
+    //     .map(([i, value]) => `((TopicParts[${i + 1}]) = '${value}')`);
+    const where = path2where(path);
+
+
+    if (where.length == 0) {
+      // underspecified, don't set TTL
+      return;
+    }
+
+    const tableExists = await this.client.query({
+      query: `SELECT name, create_table_query	FROM system.tables WHERE name = '${
+      this.mqttHistoryTable}'`,
+      format: 'JSONEachRow'
+    });
+
+    const tables = await tableExists.json();
+    const originalCreateQuery = tables[0].create_table_query;
+    const matched = originalCreateQuery.match(/TTL (.*) SETTINGS/);
+    const ttls = matched ? matched[1].split(',').map(x => x.trim()) : [];
+
+    const whereStatement = `WHERE ${where.join(' AND ')}`;
+    let present = false;
+
+    // check if TTL statement already present on table definiton
+    ttls.forEach((ttl, i) => {
+      if (ttl.endsWith(whereStatement)) {
+        // condition already present, just replace it to update time
+        ttls[i] = newTTLStatement;
+        present = true;
+      }
+    });
+
+    if (!present) {
+      ttls.push(`toDateTime(Timestamp) + toIntervalDay(${ttlDays}) ${whereStatement}`);
     }
 
-    // Subscribe to the topic
-    dataCache.subscribePath(topic, async (value, topicString) => {
-      const row = {
-        Timestamp: new Date(),
-        TopicParts: topicToPath(topicString), // topic as array
-      };
-
-      if (value !== null && value !== undefined) {
-        row.Payload = JSON.stringify(value);
-      } // else: omit
-
-      try {
-        await this.client.insert({
-          table: this.mqttHistoryTable,
-          values: [row],
-          format: 'JSONEachRow'
-        });
-      } catch (error) {
-        console.error('Error inserting MQTT message into ClickHouse:', error.message);
+    await this.client.command({
+      query: `ALTER TABLE ${this.mqttHistoryTable} MODIFY TTL ${ttls.join(',')}`,
+      clickhouse_settings: {
+        wait_end_of_query: 1,
       }
     });
   }
@@ -258,28 +341,18 @@ class ClickHouse {
       limit = 1000
     } = options;
 
-    const [OrgId, DeviceId, Scope, CapabilityName, CapabilityVersion, ...subPath]
-      = topicToPath(topicSelector);
-    // store as objects so we can refer to them by column name
-    const fields = { OrgId, DeviceId, Scope, CapabilityName, CapabilityVersion };
-
-    const selectors = ['Payload', 'TopicParts', 'Timestamp', 'SubTopic'];
-    const where = [];
+    const path = topicToPath(topicSelector);
 
     // interpret wildcards
-    _.forEach(fields, (value, field) => {
-      if (value.startsWith('+')) {
-        // it's a wild card, add to selectors
-        selectors.push(field);
-      } else {
-        // it's a constant, filter by it
-        where.push(`${field} = '${value}'`);
-      }
-    });
-
-    // special WHERE conditions for SubPath (if given)
-    subPath?.forEach((value, i) =>
-      !value.startsWith('+') && where.push(`SubTopic[${i}] = '${value}'`));
+    // const where = [];
+    // _.forEach(path, (value, i) => {
+    //   if (!['+','#'].includes(value[0])) {
+    //     // it's a constant, filter by it
+    //     where.push(`TopicParts[${i + 1}] = '${value}'`);
+    //     // Note that ClickHouse/SQL index starting at 1, not 0
+    //   }
+    // });
+    const where = path2where(path);
 
     since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
     until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);
@@ -289,7 +362,7 @@ class ClickHouse {
       : '';
 
     const result = await this.client.query({
-      query: `SELECT ${selectors.join(',')} FROM ${this.mqttHistoryTable} ${
+      query: `SELECT Payload,TopicParts,Timestamp FROM default.${this.mqttHistoryTable} ${
       whereStatement} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`,
       format: 'JSONEachRow'
     });
@@ -301,6 +374,12 @@ class ClickHouse {
     return rows.map(row => {
       row.Payload = row.Payload ? JSON.parse(row.Payload) : null;
       row.Timestamp = new Date(row.Timestamp);
+      row.OrgId = row.TopicParts[0];
+      row.DeviceId = row.TopicParts[1];
+      row.Scope = row.TopicParts[2];
+      row.CapabilityName = row.TopicParts[3];
+      row.CapabilityVersion = row.TopicParts[4];
+      row.SubTopic = row.TopicParts.slice(5);
       return row;
     });
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {
@@ -21,14 +21,13 @@
   },
   "main": "index.js",
   "scripts": {
-    "test": "mocha -w test/*.test.js -b"
+    "test": "node --test --watch"
   },
   "dependencies": {
     "@clickhouse/client": "^1.12.1",
     "@transitive-sdk/datacache": "^0.14.1"
   },
   "devDependencies": {
-    "dotenv": "^17.2.3",
-    "mocha": "^11.7.5"
+    "dotenv": "^17.2.3"
   }
 }

package/test/clickhouse.test.js

@@ -1,5 +1,6 @@
 const assert = require('assert');
 const { EventEmitter, once } = require('node:events');
+const { describe, it, before, after, beforeEach } = require('node:test');
 const dotenv = require('dotenv');
 const { DataCache } = require('@transitive-sdk/datacache');
 const { wait } = require('../../index');
@@ -30,32 +31,44 @@ const interceptInserts = () => {
 
 /** Query mqtt_history rows for a given org */
 const queryRowsByOrg = async (org, options = {}) =>
-  await clickhouse.queryMQTTHistory({ topicSelector: `/${org}/+/+/+/+/+` });
+  await clickhouse.queryMQTTHistory({
+    topicSelector: `/${org}/+/+/+/+/+`,
+    ...options
+  });
 
 /** Generate unique org ID for test isolation */
 const testOrg = (suffix) => `clickhouse_test_${suffix}_${Date.now()}`;
 
 
 describe('ClickHouse', function() {
-  this.timeout(10000);
+  // this.timeout(10000);
 
   let emitter;
+  const dataCache = new DataCache({});
 
   before(async () => {
     clickhouse.init({ url: CLICKHOUSE_URL });
     /* Register for `insert` events on ClickHouse client */
     emitter = interceptInserts();
 
-    await clickhouse.client.command({
-      query: `DROP TABLE IF EXISTS ${TABLE_NAME}`,
+    for (let query of [
+      `DROP TABLE IF EXISTS ${TABLE_NAME}`,
+      `DROP ROW POLICY IF EXISTS row_policy ON ${TABLE_NAME}`
+    ]) await clickhouse.client.command({
+      query,
       clickhouse_settings: { wait_end_of_query: 1 }
     });
+
+    await clickhouse.enableHistory({
+      dataCache,
+      tableName: TABLE_NAME
+    });
+
+    await clickhouse.registerMqttTopicForStorage(STANDARD_TOPIC_PATTERN);
   });
 
-  describe('ensureMqttHistoryTable', () => {
+  describe('enableHistory', () => {
     it('should create the mqtt_history table', async () => {
-      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 31);
-
       const result = await clickhouse.client.query({
         query: `SELECT name FROM system.tables WHERE name = '${TABLE_NAME}'`,
         format: 'JSONEachRow'
@@ -67,15 +80,9 @@ describe('ClickHouse', function() {
   });
 
   describe('registerMqttTopicForStorage', () => {
-    before(async () => {
-      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 32);
-    });
-
     it('should insert MQTT messages into ClickHouse', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('insert');
 
-      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
       dataCache.update([org, 'device1', '@myscope', 'test-cap', '1.0.0', 'data'], 42.5);
       await once(emitter, 'insert');
 
@@ -90,10 +97,8 @@ describe('ClickHouse', function() {
     });
 
     it('should store string payloads as-is', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('string');
 
-      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
       dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'msg'], 'hello world');
       await once(emitter, 'insert');
 
@@ -103,11 +108,9 @@ describe('ClickHouse', function() {
     });
 
     it('should store null values as NULL (omitted)', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('null');
-      // const done = interceptInserts(2);
 
-      clickhouse.registerMqttTopicForStorage(dataCache, '/+org/+device/#');
+      // clickhouse.registerMqttTopicForStorage('/+org/+device/#');
       dataCache.update([org, 'device1', 'data'], 'initial');
       // Small delay to ensure timestamp ordering
       await new Promise(resolve => setTimeout(resolve, 10));
@@ -123,11 +126,9 @@ describe('ClickHouse', function() {
     });
 
     it('should store object payloads as JSON', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('object');
       const payload = { sensor: 'temp', value: 25.5, nested: { a: 1 } };
 
-      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
       dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'readings'], payload);
       await once(emitter, 'insert');
 
@@ -138,10 +139,8 @@ describe('ClickHouse', function() {
     });
 
     it('should parse nested subtopics correctly', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('subtopic');
 
-      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
       dataCache.update([org, 'device1', '@myscope', 'cap', '2.0.0', 'level1', 'level2'], 'value');
       await once(emitter, 'insert');
 
@@ -153,10 +152,8 @@ describe('ClickHouse', function() {
     });
 
     it('should handle multiple updates to different subtopics', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('multi');
 
-      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
       dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'battery'], 85);
       dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'temperature'], 42);
       await once(emitter, 'insert');
@@ -172,10 +169,8 @@ describe('ClickHouse', function() {
     });
 
     it('should work with unnamed wildcards', async () => {
-      const dataCache = new DataCache({});
       const org = testOrg('unnamed');
 
-      clickhouse.registerMqttTopicForStorage(dataCache, '/+/+/+/+/+/#');
       dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'data'], { x: 1 });
       await once(emitter, 'insert');
 
@@ -184,24 +179,35 @@ describe('ClickHouse', function() {
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepStrictEqual(row.Payload, { x: 1 });
     });
+
+
+    it('should avoid duplicates', async () => {
+      const org = testOrg('multi');
+
+      // register multiple overlapping topics, want to see only one insertion
+      await clickhouse.registerMqttTopicForStorage('/+org/+/+/+/+/#');
+      await clickhouse.registerMqttTopicForStorage('/+/+device/+/+/+/#');
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'data'], { x: 1 });
+      await once(emitter, 'insert');
+      const rows = await queryRowsByOrg(org);
+      assert.equal(rows.length, 1);
+    });
+
   });
 
 
   describe('queryMQTTHistory', () => {
 
-    const dataCache = new DataCache({});
     const org = testOrg('query');
 
     before(async () => {
-      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 33);
-
       // clear
       await clickhouse.client.command({
         query: `TRUNCATE TABLE ${TABLE_NAME}`,
         clickhouse_settings: { wait_end_of_query: 1 }
       });
 
-      clickhouse.registerMqttTopicForStorage(dataCache, '#');
+      await clickhouse.registerMqttTopicForStorage('#');
       dataCache.update([org, 'device1', '@myscope', 'nullcap', '1.0.0', 'willBeNull'], 1234);
       dataCache.update([org, 'device1', '@myscope', 'capdata', '1.0.0', 'data'], { x: 1 });
       dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'data2'], { y: 2 });
@@ -237,20 +243,20 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
-        topicSelector: `/${org}/+/+/+/+/+/data2` });
+        topicSelector: `/${org}/+/+/+/+/data2` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepStrictEqual(row.Payload, { y: 2 });
     });
 
     it('queries based on sub-topic selectors with wildcards', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
-        topicSelector: `/${org}/+/+/+/+/+/+/sub2/+` });
+        topicSelector: `/${org}/+/+/+/+/+/sub2/+` });
       assert.deepStrictEqual(row.SubTopic[2], 'sub3.1');
     });
 
     it('queries based on multiple sub-topic selectors with wildcards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/${org}/+/+/+/+/+/sub1/+/+` });
+        topicSelector: `/${org}/+/+/+/+/sub1/+/+` });
       assert.strictEqual(rows[0].SubTopic.length, 3);
       assert.strictEqual(rows[0].SubTopic[2], 'sub3.1');
       assert.strictEqual(rows[1].SubTopic[2], 'sub3.2');
@@ -258,7 +264,7 @@ describe('ClickHouse', function() {
 
     it('returns the history', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/${org}/+/+/+/+/+/data/+/+` });
+        topicSelector: `/${org}/+/+/+/+/data/+/+` });
       assert.deepStrictEqual(rows.length, 2);
       assert.deepStrictEqual(rows[0].Payload, {x: 1});
       assert.deepStrictEqual(rows[1].Payload, {x: 2});
@@ -267,23 +273,20 @@ describe('ClickHouse', function() {
 
     it('handles null values', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/${org}/+/+/+/+/+/willBeNull` });
+        topicSelector: `/${org}/+/+/+/+/willBeNull` });
       assert.strictEqual(rows.at(-1).Payload, null);
     });
   });
 
   /** Test performance of the table (index). */
-  describe('performance', () => {
+  describe('performance', {timeout: 10000}, () => {
 
     const ROWS = 1_000_000; // number of rows to insert (mock)
     // time gap between inserted values (to stretch over several partitions):
     const GAP = 1_000;
-    const dataCache = new DataCache({});
     const now = Date.now();
 
     before(async () => {
-      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 33);
-
       // clear
       await clickhouse.client.exec({
         query: `TRUNCATE TABLE ${TABLE_NAME}`,
@@ -311,14 +314,9 @@ describe('ClickHouse', function() {
 
     let start;
     beforeEach(() => {
-      console.time('elapsed');
       start = performance.now();
     });
 
-    afterEach(() => {
-      console.timeEnd('elapsed');
-    });
-
     /** Assert that no more than limit ms have passed since start of test case. */
     const assertTimelimit = (limit) => {
       assert(performance.now() - start < limit, `Less than ${limit} ms`);
@@ -345,25 +343,25 @@ describe('ClickHouse', function() {
 
     it('quickly filters by DeviceId', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/device123/+/+/+/+/data`,
+        topicSelector: `/+/device123/+/+/+/data`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 1000);
-      assertTimelimit(ROWS / 10000);
+      assertTimelimit(ROWS / 1000);
     });
 
     it('quickly filters by CapabilityName', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/cap34/+/+/data`,
+        topicSelector: `/+/+/+/cap34/+/data`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 100);
-      assertTimelimit(ROWS / 10000);
+      assertTimelimit(ROWS / 1000);
     });
 
     it('quickly filters by time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/data`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });
@@ -373,7 +371,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by time: until', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/data`,
         until: new Date(now + 400 * GAP),
         limit: 2 * ROWS,
       });
@@ -383,14 +381,13 @@ describe('ClickHouse', function() {
 
     it('quickly filters by org and time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/org23/+/+/+/+/+/data`,
+        topicSelector: `/org23/+/+/+/+/data`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, 8);
       assertTimelimit(ROWS / 10000);
     });
-
   });
 
 });