@transitive-sdk/clickhouse 0.1.1 → 0.2.0

package/index.js

@@ -1,19 +1,44 @@
+const _ = require('lodash');
 const { createClient } = require('@clickhouse/client');
+const { topicToPath } = require('@transitive-sdk/datacache');
+
+// Default TTL in days for mqtt_history table
+const DEFAULT_TTL_DAYS = 30;
+
+// Shared multi-tenant schema components used by createTable and ensureMqttHistoryTable
+const MULTI_TENANT_SCHEMA = {
+  // Column definitions for OrgId and DeviceId
+  columns: [
+    'OrgId LowCardinality(String) CODEC(ZSTD(1))',
+    'DeviceId LowCardinality(String) CODEC(ZSTD(1))'
+  ],
+  // Bloom filter indexes for efficient filtering
+  indexes: [
+    'INDEX idx_orgid (OrgId) TYPE bloom_filter(0.01) GRANULARITY 1',
+    'INDEX idx_deviceid (DeviceId) TYPE bloom_filter(0.01) GRANULARITY 1'
+  ]
+};
 
 /** Singleton ClickHouse client wrapper with multi-tenant table support */
 class ClickHouse {
-  init({url, dbName, user, password} = {}) {
+
+  _client = null;
+  mqttHistoryTable = null; /// name of the table used for MQTT history, if used
+
+  /** Create the client, connecting to Clickhouse */
+  init({ url, dbName, user, password } = {}) {
     const _url = url || process.env.CLICKHOUSE_URL || 'http://clickhouse:8123';
     const _dbName = dbName || process.env.CLICKHOUSE_DB || 'default';
     const _user = user || process.env.CLICKHOUSE_USER || 'default';
-    const _password = password || process.env.CLICKHOUSE_PASSWORD || '';
-    console.debug(`Creating ClickHouse client for URL: ${_url}, DB: ${_dbName}, User: ${_user}`);
+
+    // console.debug(`Creating ClickHouse client for URL: ${_url}, DB: ${_dbName}, User: ${_user}`);
+
     this._client = createClient({
       url: _url,
       max_open_connections: 10,
       database: _dbName,
       username: _user,
-      password: _password,
+      password: password || process.env.CLICKHOUSE_PASSWORD || '',
       clickhouse_settings: {
         // https://clickhouse.com/docs/en/operations/settings/settings#async-insert
         async_insert: 1,
@@ -25,10 +50,14 @@ class ClickHouse {
         async_insert_busy_timeout_ms: 1000,
         // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
         date_time_input_format: 'best_effort',
+        // Include MATERILIZED columns in query results
+        // https://clickhouse.com/docs/operations/settings/settings#asterisk_include_materialized_columns
+        // asterisk_include_materialized_columns: 1
       },
     });
   }
 
+  /** Get the Clickhouse client (from @clickhouse/client) */
   get client() {
     if (this._client == undefined) {
       console.warn('Cannot access ClickHouse client before init() is called');
@@ -37,9 +66,8 @@ class ClickHouse {
     return this._client;
   }
 
-  /** Create a table if it does not already exist
-   * adding OrgId and DeviceId columns to the schema
-   * for multi-tenancy support.
+  /** Create a table if it does not already exist adding OrgId and DeviceId
+   * columns to the schema for multi-tenancy support.
    * @param {string} tableName - name of the table to create
    * @param {Array<string>} columns - array of column definitions and indexes, e.g. ['Timestamp DateTime CODEC(ZSTD(1))', 'Value Float32 CODEC(ZSTD(1))']
    * @param {Array<string>} settings - array of table settings, e.g. ['ENGINE = MergeTree()', 'ORDER BY (Timestamp)']
@@ -47,10 +75,8 @@ class ClickHouse {
   async createTable(tableName, columns, settings = []) {
     const fullSchema = [
       ...columns,
-      'OrgId String CODEC(ZSTD(1))',
-      'DeviceId String CODEC(ZSTD(1))',
-      'INDEX idx_orgid (OrgId) TYPE bloom_filter(0.01) GRANULARITY 1',
-      'INDEX idx_deviceid (DeviceId) TYPE bloom_filter(0.01) GRANULARITY 1'
+      ...MULTI_TENANT_SCHEMA.columns,
+      ...MULTI_TENANT_SCHEMA.indexes
     ];
     const query = `CREATE TABLE IF NOT EXISTS ${tableName} (${fullSchema.join(', ')}) ${settings.join(' ')}`;
 
@@ -86,13 +112,199 @@ class ClickHouse {
       OrgId: orgId,
       DeviceId: deviceId
     }));
+
     return await this.client.insert({
       table: tableName,
       values: rowsWithIds,
       format: 'JSONEachRow'
     });
   }
+
+  /** Update the TTL for the mqtt_history table
+   * @param {number} ttlDays - TTL in days
+   */
+  async updateMqttHistoryTTL(ttlDays) {
+    // console.log(`updating ttl to ${ttlDays}`);
+    await this.client.command({
+      query: `ALTER TABLE ${this.mqttHistoryTable} MODIFY TTL toDateTime(Timestamp) + toIntervalDay(${ttlDays})`,
+      clickhouse_settings: {
+        wait_end_of_query: 1,
+      }
+    });
+  }
+
+  /** Ensure the mqtt_history table exists with the correct schema
+   * @param {number} ttlDays - TTL in days (default: 30)
+   */
+  async ensureMqttHistoryTable(tableName = 'mqtt_history', ttlDays = DEFAULT_TTL_DAYS) {
+    if (this.mqttHistoryTable != tableName) {
+      console.warn(`creating or altering mqtt history table ${tableName}`);
+    }
+
+    const ttlExpression = `TTL toDateTime(Timestamp) + toIntervalDay(${ttlDays})`;
+
+    // Check if table already exists before creating
+    const tableExists = await this.client.query({
+      query: `SELECT name, create_table_query	FROM system.tables WHERE name = '${this.mqttHistoryTable}' AND database = currentDatabase()`,
+      format: 'JSONEachRow'
+    });
+    const tables = await tableExists.json();
+
+    if (tables.length > 0) {
+      // table already exists, verify TTL
+      const originalCreateQuery = tables[0].create_table_query;
+
+      // Update table if it differs
+      if (!originalCreateQuery.includes(ttlExpression)) {
+        await this.updateMqttHistoryTTL(ttlDays);
+      }
+
+    } else {
+      // Create the table
+
+      const columns = [
+        // High-precision event time; Delta + ZSTD is a common combo for time-series
+        'Timestamp DateTime64(6) CODEC(Delta, ZSTD(1))',
+        // Raw MQTT topic split into parts; kept as Array(String) for flexibility
+        'TopicParts Array(String) CODEC(ZSTD(1))',
+        // Org/device fields materialized from TopicParts (always computed, not overridable)
+        'OrgId LowCardinality(String) MATERIALIZED TopicParts[1] CODEC(ZSTD(1))',
+        'DeviceId LowCardinality(String) MATERIALIZED TopicParts[2] CODEC(ZSTD(1))',
+        'Scope LowCardinality(String) MATERIALIZED TopicParts[3] CODEC(ZSTD(1))',
+        'CapabilityName LowCardinality(String) MATERIALIZED TopicParts[4] CODEC(ZSTD(1))',
+        'CapabilityVersion LowCardinality(String) MATERIALIZED TopicParts[5] CODEC(ZSTD(1))',
+        // Remaining topic segments stored as an array for less-structured suffixes
+        'SubTopic Array(String) MATERIALIZED arraySlice(TopicParts, 6) CODEC(ZSTD(1))',
+        // Payload stored as a String, compressed with ZSTD(1). This allows us to
+        // store atomic values (still stringified) as opposed to only JSON objects,
+        // as the JSON type would require.
+        'Payload String CODEC(ZSTD(1))',
+        // Bloom filter indexes (shared multi-tenant indexes)
+        ...MULTI_TENANT_SCHEMA.indexes,
+        'INDEX idx_scope (Scope) TYPE bloom_filter(0.01) GRANULARITY 1',
+        'INDEX idx_capability (CapabilityName) TYPE bloom_filter(0.01) GRANULARITY 1'
+      ];
+
+      const query = `CREATE TABLE IF NOT EXISTS ${tableName} (${columns.join(', ')})
+        ENGINE = MergeTree()
+        PARTITION BY toYYYYMMDD(Timestamp)
+        ORDER BY (OrgId, toUnixTimestamp64Micro(Timestamp), TopicParts)
+        ${ttlExpression}
+        SETTINGS
+        index_granularity = 8192,
+        ttl_only_drop_parts = 1`;
+      // Note: PRIMARY KEY is not needed because we want it to be the same as
+      // ORDER BY, which is what ClickHouse does automatically.
+
+      await this.client.command({
+        query,
+        clickhouse_settings: {
+          wait_end_of_query: 1,
+        }
+      });
+    }
+
+    this.mqttHistoryTable = tableName;
+  }
+
+  /** Register an MQTT topic for storage in ClickHouse subscribes to the topic
+  * and stores incoming messages JSON.stringify'd in a ClickHouse table.
+  * Retrieve using `queryMQTTHistory`, or, when quering directly, e.g., from
+  * Grafana, use the ClickHouse built-in functionality for parsing JSON, e.g.,
+  * after inserting `{ x: 1 }` use
+  * `select JSON_VALUE(Payload, '$.x') AS x FROM default.mqtt_history`.
+  * NOTE: `ensureMqttHistoryTable` must be called before registering topics
+  * @param {Object} dataCache - DataCache instance to use for subscribing
+  * @param {string} topic - MQTT topic to register
+  */
+  registerMqttTopicForStorage(dataCache, topic) {
+    if (!this.mqttHistoryTable) {
+      throw new Error('ensureMqttHistoryTable must be called before registerMqttTopicForStorage');
+    }
+
+    // Subscribe to the topic
+    dataCache.subscribePath(topic, async (value, topicString) => {
+      const row = {
+        Timestamp: new Date(),
+        TopicParts: topicToPath(topicString), // topic as array
+      };
+
+      if (value !== null && value !== undefined) {
+        row.Payload = JSON.stringify(value);
+      } // else: omit
+
+      try {
+        await this.client.insert({
+          table: this.mqttHistoryTable,
+          values: [row],
+          format: 'JSONEachRow'
+        });
+      } catch (error) {
+        console.error('Error inserting MQTT message into ClickHouse:', error.message);
+      }
+    });
+  }
+
+  /** Query historic MQTT payloads based on topic selector (with the usual
+  * wildcards), as well as a time range. Does the inverse transform of the
+  * payload of registerMqttTopicForStorage. */
+  async queryMQTTHistory(options = {}) {
+
+    const {
+      topicSelector,
+      since = undefined,
+      until = undefined,
+      orderBy = 'Timestamp ASC',
+      limit = 1000
+    } = options;
+
+    const [OrgId, DeviceId, Scope, CapabilityName, CapabilityVersion, ...subPath]
+      = topicToPath(topicSelector);
+    // store as objects so we can refer to them by column name
+    const fields = { OrgId, DeviceId, Scope, CapabilityName, CapabilityVersion };
+
+    const selectors = ['Payload', 'TopicParts', 'Timestamp', 'SubTopic'];
+    const where = [];
+
+    // interpret wildcards
+    _.forEach(fields, (value, field) => {
+      if (value.startsWith('+')) {
+        // it's a wild card, add to selectors
+        selectors.push(field);
+      } else {
+        // it's a constant, filter by it
+        where.push(`${field} = '${value}'`);
+      }
+    });
+
+    // special WHERE conditions for SubPath (if given)
+    subPath?.forEach((value, i) =>
+      !value.startsWith('+') && where.push(`SubTopic[${i}] = '${value}'`));
+
+    since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
+    until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);
+
+    const whereStatement = where.length > 0
+      ? `WHERE ${where.join(' AND ')}`
+      : '';
+
+    const result = await this.client.query({
+      query: `SELECT ${selectors.join(',')} FROM ${this.mqttHistoryTable} ${
+      whereStatement} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`,
+      format: 'JSONEachRow'
+    });
+
+    const rows = await result.json();
+
+    // map payloads back from JSON; this is the inverse of what we do in
+    // registerMqttTopicForStorage
+    return rows.map(row => {
+      row.Payload = row.Payload ? JSON.parse(row.Payload) : null;
+      row.Timestamp = new Date(row.Timestamp);
+      return row;
+    });
+  }
 }
 
 const instance = new ClickHouse();
-module.exports = instance;
+module.exports = instance;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {
@@ -20,9 +20,15 @@
     "access": "public"
   },
   "main": "index.js",
-  "scripts": {},
+  "scripts": {
+    "test": "mocha -w test/*.test.js -b"
+  },
   "dependencies": {
-    "@clickhouse/client": "^1.12.1"
+    "@clickhouse/client": "^1.12.1",
+    "@transitive-sdk/datacache": "^0.14.1"
   },
-  "devDependencies": {}
+  "devDependencies": {
+    "dotenv": "^17.2.3",
+    "mocha": "^11.7.5"
+  }
 }

package/test/clickhouse.test.js

@@ -0,0 +1,396 @@
+const assert = require('assert');
+const { EventEmitter, once } = require('node:events');
+const dotenv = require('dotenv');
+const { DataCache } = require('@transitive-sdk/datacache');
+const { wait } = require('../../index');
+
+const clickhouse = require('../index');
+
+dotenv.config({path: '~transitive/.env'});
+const CLICKHOUSE_URL = 'http://clickhouse.localhost';
+const STANDARD_TOPIC_PATTERN = '/+org/+device/+scope/+cap/+version/#';
+
+const TABLE_NAME = 'mqtt_history_tests';
+
+/** Wrap client.insert in an event emitter so we can get notified of insert
+ * events. */
+const interceptInserts = () => {
+  const emitter = new EventEmitter();
+
+  const originalInsert = clickhouse.client.insert.bind(clickhouse.client);
+  clickhouse.client.insert = async (...args) => {
+    const result = await originalInsert(...args);
+    emitter.emit('insert');
+    return result;
+  };
+
+  return emitter;
+}
+
+
+/** Query mqtt_history rows for a given org */
+const queryRowsByOrg = async (org, options = {}) =>
+  await clickhouse.queryMQTTHistory({ topicSelector: `/${org}/+/+/+/+/+` });
+
+/** Generate unique org ID for test isolation */
+const testOrg = (suffix) => `clickhouse_test_${suffix}_${Date.now()}`;
+
+
+describe('ClickHouse', function() {
+  this.timeout(10000);
+
+  let emitter;
+
+  before(async () => {
+    clickhouse.init({ url: CLICKHOUSE_URL });
+    /* Register for `insert` events on ClickHouse client */
+    emitter = interceptInserts();
+
+    await clickhouse.client.command({
+      query: `DROP TABLE IF EXISTS ${TABLE_NAME}`,
+      clickhouse_settings: { wait_end_of_query: 1 }
+    });
+  });
+
+  describe('ensureMqttHistoryTable', () => {
+    it('should create the mqtt_history table', async () => {
+      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 31);
+
+      const result = await clickhouse.client.query({
+        query: `SELECT name FROM system.tables WHERE name = '${TABLE_NAME}'`,
+        format: 'JSONEachRow'
+      });
+      const tables = await result.json();
+
+      assert(tables.length > 0, 'mqtt_history table should exist');
+    });
+  });
+
+  describe('registerMqttTopicForStorage', () => {
+    before(async () => {
+      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 32);
+    });
+
+    it('should insert MQTT messages into ClickHouse', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('insert');
+
+      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
+      dataCache.update([org, 'device1', '@myscope', 'test-cap', '1.0.0', 'data'], 42.5);
+      await once(emitter, 'insert');
+
+      const [row] = await queryRowsByOrg(org, { limit: 1 });
+      assert(!!row);
+      assert.strictEqual(row.DeviceId, 'device1');
+      assert.strictEqual(row.Scope, '@myscope');
+      assert.strictEqual(row.CapabilityName, 'test-cap');
+      assert.strictEqual(row.CapabilityVersion, '1.0.0');
+      assert.deepStrictEqual(row.SubTopic, ['data']);
+      assert.strictEqual(row.Payload, 42.5);
+    });
+
+    it('should store string payloads as-is', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('string');
+
+      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'msg'], 'hello world');
+      await once(emitter, 'insert');
+
+      const [row] = await queryRowsByOrg(org, { limit: 1 });
+
+      assert.strictEqual(row.Payload, 'hello world');
+    });
+
+    it('should store null values as NULL (omitted)', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('null');
+      // const done = interceptInserts(2);
+
+      clickhouse.registerMqttTopicForStorage(dataCache, '/+org/+device/#');
+      dataCache.update([org, 'device1', 'data'], 'initial');
+      // Small delay to ensure timestamp ordering
+      await new Promise(resolve => setTimeout(resolve, 10));
+      dataCache.update([org, 'device1', 'data'], null);
+      await once(emitter, 'insert');
+      await once(emitter, 'insert');
+
+      const rows = await queryRowsByOrg(org);
+
+      assert.strictEqual(rows.length, 2);
+      assert.strictEqual(rows[0].Payload, 'initial');
+      assert.strictEqual(rows[1].Payload, null);
+    });
+
+    it('should store object payloads as JSON', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('object');
+      const payload = { sensor: 'temp', value: 25.5, nested: { a: 1 } };
+
+      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'readings'], payload);
+      await once(emitter, 'insert');
+
+      const [row] = await queryRowsByOrg(org, { limit: 1 });
+
+      assert.deepStrictEqual(row.Payload, payload);
+      assert.deepStrictEqual(row.SubTopic, ['readings']);
+    });
+
+    it('should parse nested subtopics correctly', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('subtopic');
+
+      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
+      dataCache.update([org, 'device1', '@myscope', 'cap', '2.0.0', 'level1', 'level2'], 'value');
+      await once(emitter, 'insert');
+
+      const [row] = await queryRowsByOrg(org, { limit: 1 });
+
+      assert.strictEqual(row.Scope, '@myscope');
+      assert.strictEqual(row.CapabilityVersion, '2.0.0');
+      assert.deepStrictEqual(row.SubTopic, ['level1', 'level2']);
+    });
+
+    it('should handle multiple updates to different subtopics', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('multi');
+
+      clickhouse.registerMqttTopicForStorage(dataCache, STANDARD_TOPIC_PATTERN);
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'battery'], 85);
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'temperature'], 42);
+      await once(emitter, 'insert');
+
+      const rows = await queryRowsByOrg(org);
+
+      assert.strictEqual(rows.length, 2);
+      const subtopics = rows.map(r => r.SubTopic[0]).sort();
+      const payloads = Object.fromEntries(rows.map(r => [r.SubTopic[0], r.Payload]));
+      assert.deepStrictEqual(subtopics, ['battery', 'temperature']);
+      assert.strictEqual(payloads['battery'], 85);
+      assert.strictEqual(payloads['temperature'], 42);
+    });
+
+    it('should work with unnamed wildcards', async () => {
+      const dataCache = new DataCache({});
+      const org = testOrg('unnamed');
+
+      clickhouse.registerMqttTopicForStorage(dataCache, '/+/+/+/+/+/#');
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'data'], { x: 1 });
+      await once(emitter, 'insert');
+
+      const [row] = await queryRowsByOrg(org, { limit: 1 });
+
+      assert.strictEqual(row.DeviceId, 'device1');
+      assert.deepStrictEqual(row.Payload, { x: 1 });
+    });
+  });
+
+
+  describe('queryMQTTHistory', () => {
+
+    const dataCache = new DataCache({});
+    const org = testOrg('query');
+
+    before(async () => {
+      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 33);
+
+      // clear
+      await clickhouse.client.command({
+        query: `TRUNCATE TABLE ${TABLE_NAME}`,
+        clickhouse_settings: { wait_end_of_query: 1 }
+      });
+
+      clickhouse.registerMqttTopicForStorage(dataCache, '#');
+      dataCache.update([org, 'device1', '@myscope', 'nullcap', '1.0.0', 'willBeNull'], 1234);
+      dataCache.update([org, 'device1', '@myscope', 'capdata', '1.0.0', 'data'], { x: 1 });
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'data2'], { y: 2 });
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0',
+        'sub1', 'sub2', 'sub3.1'],
+        { isSub: 3.1, data: {string: 'some string'} });
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0',
+        'sub1', 'sub2', 'sub3.2'],
+        { isSub: 3.3, data: {aNumber: 1234} });
+      await once(emitter, 'insert');
+      await wait(100);
+      // another value, after a delay
+      dataCache.update([org, 'device1', '@myscope', 'cap', '1.0.0', 'data'], { x: 2 });
+      dataCache.update([org, 'device1', '@myscope', 'nullcap', '1.0.0', 'willBeNull'], null);
+
+      await once(emitter, 'insert');
+    });
+
+    it('queries with wild cards', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/+/+/+` });
+      assert(rows.length > 0);
+    });
+
+    it('queries with multiple selectors', async () => {
+      const [row] = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/capdata/+/+` });
+      assert.strictEqual(row.DeviceId, 'device1');
+      assert.deepEqual(row.SubTopic, ['data']);
+      assert.deepStrictEqual(row.Payload, { x: 1 });
+    });
+
+
+    it('queries based on sub-topic selectors', async () => {
+      const [row] = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/+/+/+/data2` });
+      assert.strictEqual(row.DeviceId, 'device1');
+      assert.deepStrictEqual(row.Payload, { y: 2 });
+    });
+
+    it('queries based on sub-topic selectors with wildcards', async () => {
+      const [row] = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/+/+/+/+/sub2/+` });
+      assert.deepStrictEqual(row.SubTopic[2], 'sub3.1');
+    });
+
+    it('queries based on multiple sub-topic selectors with wildcards', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/+/+/+/sub1/+/+` });
+      assert.strictEqual(rows[0].SubTopic.length, 3);
+      assert.strictEqual(rows[0].SubTopic[2], 'sub3.1');
+      assert.strictEqual(rows[1].SubTopic[2], 'sub3.2');
+    });
+
+    it('returns the history', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/+/+/+/data/+/+` });
+      assert.deepStrictEqual(rows.length, 2);
+      assert.deepStrictEqual(rows[0].Payload, {x: 1});
+      assert.deepStrictEqual(rows[1].Payload, {x: 2});
+      assert(rows[0].Timestamp < rows[1].Timestamp);
+    });
+
+    it('handles null values', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/${org}/+/+/+/+/+/willBeNull` });
+      assert.strictEqual(rows.at(-1).Payload, null);
+    });
+  });
+
+  /** Test performance of the table (index). */
+  describe('performance', () => {
+
+    const ROWS = 1_000_000; // number of rows to insert (mock)
+    // time gap between inserted values (to stretch over several partitions):
+    const GAP = 1_000;
+    const dataCache = new DataCache({});
+    const now = Date.now();
+
+    before(async () => {
+      await clickhouse.ensureMqttHistoryTable(TABLE_NAME, 33);
+
+      // clear
+      await clickhouse.client.exec({
+        query: `TRUNCATE TABLE ${TABLE_NAME}`,
+        clickhouse_settings: { wait_end_of_query: 1 }
+      });
+
+      const rows = [];
+      for (let i = 0; i < ROWS; i++) {
+       rows.push({
+          Timestamp: new Date(now + i * GAP), // use current date to avoid immediate TTL cleanup
+          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope', `cap${i % 100}`, `1.${i % 100}.0`, 'data', i],
+          Payload: { i },
+       })
+      }
+
+      await clickhouse.client.insert({
+        table: TABLE_NAME,
+        values: [rows],
+        format: 'JSONEachRow',
+        clickhouse_settings: { wait_end_of_query: 1 }
+      });
+
+      console.log(`inserted ${rows.length} rows into ${TABLE_NAME}`);
+    });
+
+    let start;
+    beforeEach(() => {
+      console.time('elapsed');
+      start = performance.now();
+    });
+
+    afterEach(() => {
+      console.timeEnd('elapsed');
+    });
+
+    /** Assert that no more than limit ms have passed since start of test case. */
+    const assertTimelimit = (limit) => {
+      assert(performance.now() - start < limit, `Less than ${limit} ms`);
+    }
+
+    it('returns the entire history in reasonable time', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/+/+/+`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS);
+      assert(rows[0].Timestamp < rows[1].Timestamp);
+      assertTimelimit(ROWS / 100);
+    });
+
+    it('quickly filters by OrgId', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org42/+/+/+/+/+`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / 50);
+      assertTimelimit(ROWS / 1000);
+    });
+
+    it('quickly filters by DeviceId', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/device123/+/+/+/+/data`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / 1000);
+      assertTimelimit(ROWS / 10000);
+    });
+
+    it('quickly filters by CapabilityName', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/cap34/+/+/data`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / 100);
+      assertTimelimit(ROWS / 10000);
+    });
+
+    it('quickly filters by time: since', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/+/+/+/data`,
+        since: new Date(now + (ROWS - 400) * GAP),
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, 400);
+      assertTimelimit(ROWS / 10000);
+    });
+
+    it('quickly filters by time: until', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/+/+/+/data`,
+        until: new Date(now + 400 * GAP),
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, 401);
+      assertTimelimit(ROWS / 10000);
+    });
+
+    it('quickly filters by org and time: since', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org23/+/+/+/+/+/data`,
+        since: new Date(now + (ROWS - 400) * GAP),
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, 8);
+      assertTimelimit(ROWS / 10000);
+    });
+
+  });
+
+});