@transitive-sdk/clickhouse 0.3.7 → 0.4.0

package/index.js

@@ -26,14 +26,17 @@ const MULTI_TENANT_SCHEMA = {
  * WHERE clause. */
 const path2where = (path) => {
   const where = [];
+  const wildIndices = [];
   _.forEach(path, (value, i) => {
     if (!['+','#'].includes(value[0])) {
       // it's a constant, filter by it
       where.push(`TopicParts[${i + 1}] = '${value}'`);
       // Note that ClickHouse/SQL index starting at 1, not 0
+    } else {
+      wildIndices.push(i);
     }
   });
-  return where;
+  return {where, wildIndices};
 };
 
 
@@ -297,7 +300,7 @@ class ClickHouse {
     // Set/update TTL for this capability and sub-topic
 
     // Derive WHERE conditions for TTL expression from non-wildcards
-    const where = path2where(path);
+    const { where } = path2where(path);
 
     if (where.length == 0) {
       // underspecified, don't set TTL
@@ -348,35 +351,81 @@ class ClickHouse {
       topicSelector,
       since = undefined,
       until = undefined,
-      orderBy = 'Timestamp ASC',
-      limit = 1000
+      // if provided, extract this sub-value of the payload-json, requires type
+      path = undefined,
+      // type of element to extract using `path`: for available types, see https://clickhouse.com/docs/sql-reference/data-types
+      type = 'String',
+      orderBy = 'time DESC',
+      limit = 1000, // end result limit (i.e., after grouping)
+
+      bins = undefined, // into how many bins to aggregate (if given, requires since)
+      // Aggregation function to use (if aggSeconds or bins is given)
+      // if `bins` or `aggregateSeconds` is given, which operator to use to compute
+      // aggregate value. Default is `count` (which works for any data type).
+      // See https://clickhouse.com/docs/sql-reference/aggregate-functions/reference.
+      agg = 'count',
     } = options;
 
-    const path = topicToPath(topicSelector);
+    let {
+      // how many seconds to group together (alternative to bins + time interval)
+      aggSeconds
+    } = options;
 
-    // interpret wildcards
-    // const where = [];
-    // _.forEach(path, (value, i) => {
-    //   if (!['+','#'].includes(value[0])) {
-    //     // it's a constant, filter by it
-    //     where.push(`TopicParts[${i + 1}] = '${value}'`);
-    //     // Note that ClickHouse/SQL index starting at 1, not 0
-    //   }
-    // });
-    const where = path2where(path);
+    /* some useful queries we'd like to support:
+
+    # get avg `i` value for each minute of the last hour (limit: 60)
+    ```sql
+    select toStartOfInterval(Timestamp, INTERVAL 60 SECOND) as time,
+      avg(JSONExtractInt(Payload,'i')) as agg
+    from mqtt_history_tests
+    GROUP BY (time)
+    ORDER BY time
+    LIMIT 60
+    ```
+    ->
+    ```js
+    { aggregateSeconds: 60, path: ['i'], type: 'Int', agg: 'avg', limit: 60 }
+    ```
+    */
+
+    const pathSelector = topicToPath(topicSelector);
 
+    // interpret wildcards
+    const { where, wildIndices } = path2where(pathSelector);
     since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
     until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);
+    const whereStatement = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
 
-    const whereStatement = where.length > 0
-      ? `WHERE ${where.join(' AND ')}`
-      : '';
+    const extractValue = path && type
+      ? `JSONExtract(Payload, ${path.map(s => `'${s}'`).join(', ')}, '${type}')`
+      : 'Payload';
 
-    const result = await this.client.query({
-      query: `SELECT Payload,TopicParts,Timestamp FROM default.${this.mqttHistoryTable} ${
-      whereStatement} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`,
-      format: 'JSONEachRow'
-    });
+    let select = [`${extractValue} as value`, 'Payload', 'TopicParts',
+      'Timestamp', 'Timestamp as time'];
+
+    let group = '';
+    if (bins > 1 && since) {
+      // compute aggSeconds from desired number of bins and `since`
+      const duration = (until || Date.now()) - since.getTime();
+      aggSeconds = Math.floor((duration/1000)/(bins - 1));
+    }
+
+    // if aggregation is requested, build the GROUP BY expression and update SELECT
+    if (aggSeconds) {
+      // SQL sub-string to extract the desired value from the JSON payload
+      const wildParts = wildIndices.map(i => `TopicParts[${i + 1}]`);
+      // update SELECT statement with aggregations
+      select = [`${agg}(${extractValue}) as aggValue`,
+        ...wildParts,
+        `toStartOfInterval(Timestamp, INTERVAL ${aggSeconds} SECOND) as time`
+      ];
+      group = `GROUP BY (time,${wildParts.join(',')})`
+    }
+
+    const query = `SELECT ${select.join(',')} FROM default.${this.mqttHistoryTable} ${
+      whereStatement} ${group} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`;
+    // console.log(query);
+    const result = await this.client.query({ query, format: 'JSONEachRow' });
 
     const rows = await result.json();
 
@@ -385,12 +434,12 @@ class ClickHouse {
     return rows.map(row => {
       row.Payload = row.Payload ? JSON.parse(row.Payload) : null;
       row.Timestamp = new Date(row.Timestamp);
-      row.OrgId = row.TopicParts[0];
-      row.DeviceId = row.TopicParts[1];
-      row.Scope = row.TopicParts[2];
-      row.CapabilityName = row.TopicParts[3];
-      row.CapabilityVersion = row.TopicParts[4];
-      row.SubTopic = row.TopicParts.slice(5);
+      row.OrgId = row.TopicParts?.[0];
+      row.DeviceId = row.TopicParts?.[1];
+      row.Scope = row.TopicParts?.[2];
+      row.CapabilityName = row.TopicParts?.[3];
+      row.CapabilityVersion = row.TopicParts?.[4];
+      row.SubTopic = row.TopicParts?.slice(5);
       return row;
     });
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.3.7",
+  "version": "0.4.0",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {

package/test/clickhouse.test.js

@@ -33,6 +33,7 @@ const interceptInserts = () => {
 const queryRowsByOrg = async (org, options = {}) =>
   await clickhouse.queryMQTTHistory({
     topicSelector: `/${org}/+/+/+/+/+`,
+    orderBy: 'Timestamp ASC',
     ...options
   });
 
@@ -245,12 +246,14 @@ describe('ClickHouse', function() {
 
     it('queries with wild cards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/+` });
       assert(rows.length > 0);
     });
 
     it('queries with multiple selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/capdata/+/+` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepEqual(row.SubTopic, ['data']);
@@ -260,6 +263,7 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/data2` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepStrictEqual(row.Payload, { y: 2 });
@@ -267,12 +271,14 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors with wildcards', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/+/sub2/+` });
       assert.deepStrictEqual(row.SubTopic[2], 'sub3.1');
     });
 
     it('queries based on multiple sub-topic selectors with wildcards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/sub1/+/+` });
       assert.strictEqual(rows[0].SubTopic.length, 3);
       assert.strictEqual(rows[0].SubTopic[2], 'sub3.1');
@@ -281,6 +287,7 @@ describe('ClickHouse', function() {
 
     it('returns the history', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/data/+/+` });
       assert.deepStrictEqual(rows.length, 2);
       assert.deepStrictEqual(rows[0].Payload, {x: 1});
@@ -290,9 +297,20 @@ describe('ClickHouse', function() {
 
     it('handles null values', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/willBeNull` });
       assert.strictEqual(rows.at(-1).Payload, null);
     });
+
+    it('extracts sub-values', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
+        topicSelector: `/${org}/device1/@myscope/cap/+/sub1/sub2/sub3.2`,
+        path: ['data', 'aNumber'],
+        type: 'int'
+      });
+      assert.strictEqual(rows[0].value, 1234);
+    });
   });
 
   /** Test performance of the table (index). */
@@ -314,7 +332,8 @@ describe('ClickHouse', function() {
       for (let i = 0; i < ROWS; i++) {
        rows.push({
           Timestamp: new Date(now + i * GAP), // use current date to avoid immediate TTL cleanup
-          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope', `cap${i % 100}`, `1.${i % 100}.0`, 'data', i],
+          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope',
+            `cap${i % 100}`, `1.${i % 100}.0`, `data_${i % 1000}`],
           Payload: { i },
        })
       }
@@ -345,7 +364,7 @@ describe('ClickHouse', function() {
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS);
-      assert(rows[0].Timestamp < rows[1].Timestamp);
+      assert(rows[0].Timestamp > rows[1].Timestamp);
       assertTimelimit(ROWS / 100);
     });
 
@@ -360,7 +379,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by DeviceId', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/device123/+/+/+/data`,
+        topicSelector: `/+/device123/+/+/+/+`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 1000);
@@ -369,16 +388,25 @@ describe('ClickHouse', function() {
 
     it('quickly filters by CapabilityName', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/cap34/+/data`,
+        topicSelector: `/+/+/+/cap34/+/+`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 100);
       assertTimelimit(ROWS / 1000);
     });
 
+    it('quickly filters by SubTopic', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/+/+/data_123`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / 1000);
+      assertTimelimit(ROWS / 1000);
+    });
+
     it('quickly filters by time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/+`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });
@@ -388,7 +416,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by time: until', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/+`,
         until: new Date(now + 400 * GAP),
         limit: 2 * ROWS,
       });
@@ -398,13 +426,52 @@ describe('ClickHouse', function() {
 
     it('quickly filters by org and time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/org23/+/+/+/+/data`,
+        topicSelector: `/org23/+/+/+/+/+`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, 8);
       assertTimelimit(ROWS / 10000);
     });
-  });
 
+    it('quickly filters and aggregates by time', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org0/device0/@myscope/cap0/1.0.0/data_0`,
+        since: new Date(now),
+        until: new Date(now + ROWS * GAP),
+        bins: 60,
+        limit: 2 * ROWS,
+      });
+      // there can be one-off errors due to rounding down to start of interval:
+      assert(Math.abs(rows.length - 60) < 2);
+      assertTimelimit(ROWS / 10000);
+    });
+
+    it('quickly filters, aggregates by time, extracts value, and averages', async () => {
+      const aggSeconds = 1000;
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org0/device0/@myscope/cap0/1.0.0/data_0`,
+        aggSeconds,
+        path: ['i'],
+        type: 'int',
+        agg: 'avg',
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / aggSeconds);
+      assertTimelimit(ROWS / 1000);
+    });
+
+    it('quickly filters, aggregates by time, extracts value, and averages, per device and sub-value', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org0/+/@myscope/cap0/1.0.0/+`,
+        aggSeconds: 1000,
+        path: ['i'],
+        type: 'int',
+        agg: 'avg',
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, 10000);
+      assertTimelimit(ROWS / 1000);
+    });
+  });
 });