@transitive-sdk/clickhouse 0.3.8 → 0.4.1

package/index.js

@@ -26,14 +26,17 @@ const MULTI_TENANT_SCHEMA = {
  * WHERE clause. */
 const path2where = (path) => {
   const where = [];
+  const wildIndices = [];
   _.forEach(path, (value, i) => {
     if (!['+','#'].includes(value[0])) {
       // it's a constant, filter by it
       where.push(`TopicParts[${i + 1}] = '${value}'`);
       // Note that ClickHouse/SQL index starting at 1, not 0
+    } else {
+      wildIndices.push(i);
     }
   });
-  return where;
+  return {where, wildIndices};
 };
 
 
@@ -297,7 +300,7 @@ class ClickHouse {
     // Set/update TTL for this capability and sub-topic
 
     // Derive WHERE conditions for TTL expression from non-wildcards
-    const where = path2where(path);
+    const { where } = path2where(path);
 
     if (where.length == 0) {
       // underspecified, don't set TTL
@@ -348,23 +351,83 @@ class ClickHouse {
       topicSelector,
       since = undefined,
       until = undefined,
-      orderBy = 'Timestamp DESC',
-      limit = 1000
+      // if provided, extract this sub-value of the payload-json, requires type
+      path = undefined,
+      // type of element to extract using `path`: for available types, see https://clickhouse.com/docs/sql-reference/data-types
+      type = 'String',
+      orderBy = 'time DESC',
+      limit = 1000, // end result limit (i.e., after grouping)
+
+      bins = undefined, // into how many bins to aggregate (if given, requires since)
+      // Aggregation function to use (if aggSeconds or bins is given)
+      // if `bins` or `aggregateSeconds` is given, which operator to use to compute
+      // aggregate value. Default is `count` (which works for any data type).
+      // See https://clickhouse.com/docs/sql-reference/aggregate-functions/reference.
+      agg = 'count',
+    } = options;
+
+    let {
+      // how many seconds to group together (alternative to bins + time interval)
+      aggSeconds
     } = options;
 
-    const path = topicToPath(topicSelector);
+    /* some useful queries we'd like to support:
+
+    # get avg `i` value for each minute of the last hour (limit: 60)
+    ```sql
+    select toStartOfInterval(Timestamp, INTERVAL 60 SECOND) as time,
+      avg(JSONExtractInt(Payload,'i')) as agg
+    from mqtt_history_tests
+    GROUP BY (time)
+    ORDER BY time
+    LIMIT 60
+    ```
+    ->
+    ```js
+    { aggregateSeconds: 60, path: ['i'], type: 'Int', agg: 'avg', limit: 60 }
+    ```
+    */
+
+    const pathSelector = topicToPath(topicSelector);
 
     // interpret wildcards
-    const where = path2where(path);
+    const { where } = path2where(pathSelector);
     since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
     until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);
     const whereStatement = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
 
-    const result = await this.client.query({
-      query: `SELECT Payload,TopicParts,Timestamp FROM default.${this.mqttHistoryTable} ${
-      whereStatement} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`,
-      format: 'JSONEachRow'
-    });
+    const extractValue = path && type
+      ? `JSONExtract(Payload, ${path.map(s => `'${s}'`).join(', ')}, '${type}')`
+      : 'Payload';
+
+    let select = [`${extractValue} as value`, 'Payload', 'TopicParts',
+      'Timestamp', 'Timestamp as time'];
+
+    let group = '';
+    if (bins > 1 && since) {
+      // compute aggSeconds from desired number of bins and `since`
+      const duration = (until || Date.now()) - since.getTime();
+      aggSeconds = Math.floor((duration/1000)/(bins - 1));
+    }
+
+    // if aggregation is requested, build the GROUP BY expression and update SELECT
+    if (aggSeconds) {
+      // SQL sub-string to extract the desired value from the JSON payload
+      // const wildParts = wildIndices.map(i => `TopicParts[${i + 1}]`);
+      // update SELECT statement with aggregations
+      select = [`${agg}(${extractValue}) as aggValue`,
+        // ...wildParts,
+        'TopicParts',
+        `toStartOfInterval(Timestamp, INTERVAL ${aggSeconds} SECOND) as time`
+      ];
+      // group = `GROUP BY (time,${wildParts.join(',')})`
+      group = `GROUP BY (time,TopicParts)`
+    }
+
+    const query = `SELECT ${select.join(',')} FROM default.${this.mqttHistoryTable} ${
+      whereStatement} ${group} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`;
+    // console.log(query);
+    const result = await this.client.query({ query, format: 'JSONEachRow' });
 
     const rows = await result.json();
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.3.8",
+  "version": "0.4.1",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {

package/test/clickhouse.test.js

@@ -301,6 +301,16 @@ describe('ClickHouse', function() {
         topicSelector: `/${org}/+/+/+/+/willBeNull` });
       assert.strictEqual(rows.at(-1).Payload, null);
     });
+
+    it('extracts sub-values', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
+        topicSelector: `/${org}/device1/@myscope/cap/+/sub1/sub2/sub3.2`,
+        path: ['data', 'aNumber'],
+        type: 'int'
+      });
+      assert.strictEqual(rows[0].value, 1234);
+    });
   });
 
   /** Test performance of the table (index). */
@@ -323,7 +333,7 @@ describe('ClickHouse', function() {
        rows.push({
           Timestamp: new Date(now + i * GAP), // use current date to avoid immediate TTL cleanup
           TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope',
-            `cap${i % 100}`, `1.${i % 100}.0`, `data_${i % 1000}`, i],
+            `cap${i % 100}`, `1.${i % 100}.0`, `data_${i % 1000}`],
           Payload: { i },
        })
       }
@@ -423,6 +433,45 @@ describe('ClickHouse', function() {
       assert.equal(rows.length, 8);
       assertTimelimit(ROWS / 10000);
     });
-  });
 
+    it('quickly filters and aggregates by time', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org0/device0/@myscope/cap0/1.0.0/data_0`,
+        since: new Date(now),
+        until: new Date(now + ROWS * GAP),
+        bins: 60,
+        limit: 2 * ROWS,
+      });
+      // there can be one-off errors due to rounding down to start of interval:
+      assert(Math.abs(rows.length - 60) < 2);
+      assertTimelimit(ROWS / 10000);
+    });
+
+    it('quickly filters, aggregates by time, extracts value, and averages', async () => {
+      const aggSeconds = 1000;
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org0/device0/@myscope/cap0/1.0.0/data_0`,
+        aggSeconds,
+        path: ['i'],
+        type: 'int',
+        agg: 'avg',
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / aggSeconds);
+      assertTimelimit(ROWS / 1000);
+    });
+
+    it('quickly filters, aggregates by time, extracts value, and averages, per device and sub-value', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/org0/+/@myscope/cap0/1.0.0/+`,
+        aggSeconds: 1000,
+        path: ['i'],
+        type: 'int',
+        agg: 'avg',
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, 10000);
+      assertTimelimit(ROWS / 1000);
+    });
+  });
 });