@transitive-sdk/clickhouse 0.4.2 → 0.5.0

package/index.js

@@ -47,6 +47,8 @@ class ClickHouse {
 
   mqttHistoryTable = null; // name of the table used for MQTT history, if used
   topics = {}; // list of topics registered for storage, as object for de-duplication
+  rowCache = {}; // cache of rows awaiting insertion, by table
+  insertionInterval = null; // the actual interval
 
   /** Create the client, connecting to Clickhouse */
   async init({ url, dbName, user, password } = {}) {
@@ -142,7 +144,7 @@ class ClickHouse {
    * @param {string} orgId - organization ID to add to each row
    * @param {string} deviceId - device ID to add to each row
    */
-  async insert(tableName, rows, orgId, deviceId) {
+  insert(tableName, rows, orgId, deviceId) {
     // assert that orgId and deviceId are provided
     if (!orgId || !deviceId) {
       throw new Error('Both orgId and deviceId must be provided for multi-tenant insert');
@@ -155,18 +157,15 @@ class ClickHouse {
       DeviceId: deviceId
     }));
 
-    return await this.client.insert({
-      table: tableName,
-      values: rowsWithIds,
-      format: 'JSONEachRow'
-    });
+    return this.addToCache(tableName, rowsWithIds);
   }
 
   /* Enable history recording. Ensure the mqtt_history table exists with the
    * correct schema, set dataCache, and subscribe to changes.
    * @param {object} options = {dataCache, tableName, ttlDays}
+   * @param {number} interval = ms interval between batch insertions
    */
-  async enableHistory(options) {
+  async enableHistory(options, interval = 10_000) {
     const { dataCache, tableName = 'mqtt_history' } = options;
 
     if (this.mqttHistoryTable != tableName) {
@@ -241,7 +240,7 @@ class ClickHouse {
       // it matches any of the registered topics (this avoid duplicate triggers),
       // then store to ClickHouse with current timestamp.
       dataCache.subscribe((changes) => {
-        _.forEach(changes, async (value, topic) => {
+        _.forEach(changes, (value, topic) => {
 
           const matched =
             _.some(this.topics, (_true, selector) => topicMatch(selector, topic));
@@ -257,24 +256,46 @@ class ClickHouse {
             row.Payload = JSON.stringify(value);
           } // else: omit
 
-          try {
-            await this.client.insert({
-              table: this.mqttHistoryTable,
-              values: [row],
-              format: 'JSONEachRow'
-            });
-          } catch (error) {
-            console.error('Error inserting MQTT message into ClickHouse:', error.message);
-          }
+          // cache it for batch insertion
+          this.addToCache(this.mqttHistoryTable, [row]);
         })
       });
+    }
 
+    this.mqttHistoryTable = tableName;
 
+    // start interval for batch insertion
+    if (!this.insertionInterval) {
+      this.insertionInterval =
+        setInterval(this.batchInsertCache.bind(this), interval);
     }
+  }
 
-    this.mqttHistoryTable = tableName;
+  /** Add the given rows to the cache for batch-insertion to the given table */
+  addToCache(table, rows) {
+    this.rowCache[this.mqttHistoryTable] ||= [];
+    this.rowCache[this.mqttHistoryTable].push(...rows);
   }
 
+  /** Function responsible fgor inserting all cached rows */
+  batchInsertCache() {
+    _.forEach(this.rowCache, (rows, table) => {
+      if (rows.length == 0) return;
+
+      try {
+        this.rowCache[table] = [];
+        this.client.insert({
+          table,
+          values: rows,
+          format: 'JSONEachRow'
+        });
+      } catch (error) {
+        console.error(`Error inserting ${rows.length} rows into ${table}`, error.message);
+      }
+    });
+  }
+
+
   /* Register an MQTT topic for storage in ClickHouse subscribes to the topic
   * and stores incoming messages JSON.stringify'd in a ClickHouse table.
   * Retrieve using `queryMQTTHistory`, or, when quering directly, e.g., from
@@ -415,7 +436,12 @@ class ClickHouse {
       // SQL sub-string to extract the desired value from the JSON payload
       // const wildParts = wildIndices.map(i => `TopicParts[${i + 1}]`);
       // update SELECT statement with aggregations
-      select = [`${agg}(${extractValue}) as aggValue`,
+
+      select = [
+        // Cast `count` result to Float64 to avoid UInt64 which ClickHouse turns
+        // into a string in JSON.
+        agg == 'count' ? `CAST(${agg}(${extractValue}), 'Float64') as aggValue`
+        : `${agg}(${extractValue}) as aggValue`,
         // ...wildParts,
         'TopicParts',
         `toStartOfInterval(Timestamp, INTERVAL ${aggSeconds} SECOND) as time`
@@ -428,7 +454,6 @@ class ClickHouse {
       whereStatement} ${group} ORDER BY ${orderBy} ${limit ? ` LIMIT ${limit}` : ''}`;
     // console.log(query);
     const result = await this.client.query({ query, format: 'JSONEachRow' });
-
     const rows = await result.json();
 
     // map payloads back from JSON; this is the inverse of what we do in

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {
@@ -24,7 +24,7 @@
     "test": "node --test --watch"
   },
   "dependencies": {
-    "@clickhouse/client": "^1.12.1",
+    "@clickhouse/client": "^1.16.0",
     "@transitive-sdk/datacache": "^0.14.1",
     "wait-port": "^1.1.0"
   },

package/test/clickhouse.test.js

@@ -63,7 +63,7 @@ describe('ClickHouse', function() {
     await clickhouse.enableHistory({
       dataCache,
       tableName: TABLE_NAME
-    });
+    }, 100);
 
     await clickhouse.registerMqttTopicForStorage(STANDARD_TOPIC_PATTERN);
   });
@@ -130,7 +130,6 @@ describe('ClickHouse', function() {
       await new Promise(resolve => setTimeout(resolve, 10));
       dataCache.update([org, 'device1', 'data'], null);
       await once(emitter, 'insert');
-      await once(emitter, 'insert');
 
       const rows = await queryRowsByOrg(org);
 
@@ -443,6 +442,7 @@ describe('ClickHouse', function() {
         limit: 2 * ROWS,
       });
       // there can be one-off errors due to rounding down to start of interval:
+      assert.strictEqual(typeof rows[0].aggValue, 'number');
       assert(Math.abs(rows.length - 60) < 2);
       assertTimelimit(ROWS / 10000);
     });
@@ -471,6 +471,7 @@ describe('ClickHouse', function() {
         limit: 2 * ROWS,
       });
       console.log(rows[0]);
+      assert.strictEqual(typeof rows[0].aggValue, 'number');
       assert.equal(rows.length, 10000);
       assertTimelimit(ROWS / 1000);
     });