@transitive-sdk/clickhouse 0.3.6 → 0.3.8

package/index.js

@@ -8,7 +8,7 @@ const { topicToPath, topicMatch } = require('@transitive-sdk/datacache');
 // Default TTL in days for mqtt_history table
 const DEFAULT_TTL_DAYS = 30;
 
-// Shared multi-tenant schema components used by createTable and ensureMqttHistoryTable
+// Shared multi-tenant schema components used by createTable and enableHistory
 const MULTI_TENANT_SCHEMA = {
   // Column definitions for OrgId and DeviceId
   columns: [
@@ -109,13 +109,15 @@ class ClickHouse {
    * @param {Array<string>} columns - array of column definitions and indexes, e.g. ['Timestamp DateTime CODEC(ZSTD(1))', 'Value Float32 CODEC(ZSTD(1))']
    * @param {Array<string>} settings - array of table settings, e.g. ['ENGINE = MergeTree()', 'ORDER BY (Timestamp)']
    */
-  async createTable(tableName, columns, settings = []) {
+  async createTable(tableName, columns, settings = ['ORDER BY (OrgId, DeviceId)']) {
     const fullSchema = [
       ...columns,
       ...MULTI_TENANT_SCHEMA.columns,
       ...MULTI_TENANT_SCHEMA.indexes
     ];
-    const query = `CREATE TABLE IF NOT EXISTS ${tableName} (${fullSchema.join(', ')}) ${settings.join(' ')}`;
+    const query = `CREATE TABLE IF NOT EXISTS ${tableName}
+      (${fullSchema.join(', ')})
+      ${settings.join(' ')}`;
 
     try {
       return await this.client.exec({
@@ -346,29 +348,17 @@ class ClickHouse {
       topicSelector,
       since = undefined,
       until = undefined,
-      orderBy = 'Timestamp ASC',
+      orderBy = 'Timestamp DESC',
       limit = 1000
     } = options;
 
     const path = topicToPath(topicSelector);
 
     // interpret wildcards
-    // const where = [];
-    // _.forEach(path, (value, i) => {
-    //   if (!['+','#'].includes(value[0])) {
-    //     // it's a constant, filter by it
-    //     where.push(`TopicParts[${i + 1}] = '${value}'`);
-    //     // Note that ClickHouse/SQL index starting at 1, not 0
-    //   }
-    // });
     const where = path2where(path);
-
     since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
     until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);
-
-    const whereStatement = where.length > 0
-      ? `WHERE ${where.join(' AND ')}`
-      : '';
+    const whereStatement = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
 
     const result = await this.client.query({
       query: `SELECT Payload,TopicParts,Timestamp FROM default.${this.mqttHistoryTable} ${

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.3.6",
+  "version": "0.3.8",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {

package/test/clickhouse.test.js

@@ -33,6 +33,7 @@ const interceptInserts = () => {
 const queryRowsByOrg = async (org, options = {}) =>
   await clickhouse.queryMQTTHistory({
     topicSelector: `/${org}/+/+/+/+/+`,
+    orderBy: 'Timestamp ASC',
     ...options
   });
 
@@ -67,6 +68,19 @@ describe('ClickHouse', function() {
     await clickhouse.registerMqttTopicForStorage(STANDARD_TOPIC_PATTERN);
   });
 
+  describe('basics', () => {
+    it('creates tables without crashing', async () => {
+      const result = await clickhouse.createTable('test_tmp',
+        ['text String']);
+
+      // clean up
+      await clickhouse.client.command({
+        query: `DROP TABLE IF EXISTS test_tmp`,
+        clickhouse_settings: { wait_end_of_query: 1 }
+      })
+    });
+  });
+
   describe('enableHistory', () => {
     it('should create the mqtt_history table', async () => {
       const result = await clickhouse.client.query({
@@ -232,12 +246,14 @@ describe('ClickHouse', function() {
 
     it('queries with wild cards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/+` });
       assert(rows.length > 0);
     });
 
     it('queries with multiple selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/capdata/+/+` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepEqual(row.SubTopic, ['data']);
@@ -247,6 +263,7 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/data2` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepStrictEqual(row.Payload, { y: 2 });
@@ -254,12 +271,14 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors with wildcards', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/+/sub2/+` });
       assert.deepStrictEqual(row.SubTopic[2], 'sub3.1');
     });
 
     it('queries based on multiple sub-topic selectors with wildcards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/sub1/+/+` });
       assert.strictEqual(rows[0].SubTopic.length, 3);
       assert.strictEqual(rows[0].SubTopic[2], 'sub3.1');
@@ -268,6 +287,7 @@ describe('ClickHouse', function() {
 
     it('returns the history', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/data/+/+` });
       assert.deepStrictEqual(rows.length, 2);
       assert.deepStrictEqual(rows[0].Payload, {x: 1});
@@ -277,6 +297,7 @@ describe('ClickHouse', function() {
 
     it('handles null values', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/willBeNull` });
       assert.strictEqual(rows.at(-1).Payload, null);
     });
@@ -301,7 +322,8 @@ describe('ClickHouse', function() {
       for (let i = 0; i < ROWS; i++) {
        rows.push({
           Timestamp: new Date(now + i * GAP), // use current date to avoid immediate TTL cleanup
-          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope', `cap${i % 100}`, `1.${i % 100}.0`, 'data', i],
+          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope',
+            `cap${i % 100}`, `1.${i % 100}.0`, `data_${i % 1000}`, i],
           Payload: { i },
        })
       }
@@ -332,7 +354,7 @@ describe('ClickHouse', function() {
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS);
-      assert(rows[0].Timestamp < rows[1].Timestamp);
+      assert(rows[0].Timestamp > rows[1].Timestamp);
       assertTimelimit(ROWS / 100);
     });
 
@@ -347,7 +369,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by DeviceId', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/device123/+/+/+/data`,
+        topicSelector: `/+/device123/+/+/+/+`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 1000);
@@ -356,16 +378,25 @@ describe('ClickHouse', function() {
 
     it('quickly filters by CapabilityName', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/cap34/+/data`,
+        topicSelector: `/+/+/+/cap34/+/+`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 100);
       assertTimelimit(ROWS / 1000);
     });
 
+    it('quickly filters by SubTopic', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/+/+/data_123`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / 1000);
+      assertTimelimit(ROWS / 1000);
+    });
+
     it('quickly filters by time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/+`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });
@@ -375,7 +406,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by time: until', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/+`,
         until: new Date(now + 400 * GAP),
         limit: 2 * ROWS,
       });
@@ -385,7 +416,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by org and time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/org23/+/+/+/+/data`,
+        topicSelector: `/org23/+/+/+/+/+`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });