@transitive-sdk/clickhouse 0.4.3 → 0.5.1

package/index.js

@@ -47,21 +47,22 @@ class ClickHouse {
 
   mqttHistoryTable = null; // name of the table used for MQTT history, if used
   topics = {}; // list of topics registered for storage, as object for de-duplication
+  rowCache = {}; // cache of rows awaiting insertion, by table
 
   /** Create the client, connecting to Clickhouse */
-  async init({ url, dbName, user, password } = {}) {
+  async init({ url, dbName, user, password, interval } = {}) {
 
     const _url = url || process.env.CLICKHOUSE_URL || 'http://clickhouse:8123';
     const _dbName = dbName || process.env.CLICKHOUSE_DB || 'default';
     const _user = user || process.env.CLICKHOUSE_USER || 'default';
+    interval ||= 10_000;
 
     const {hostname, port} = URL.parse(_url);
-    const interval = 200;
-    await waitPort({ host: hostname, port: Number(port || 80), interval }, 10000);
+    const waitInterval = 200;
+    await waitPort({ host: hostname, port: Number(port || 80), waitInterval },
+      10000);
     await new Promise(done => setTimeout(done, 200));
 
-    // console.debug(`Creating ClickHouse client for URL: ${_url}, DB: ${_dbName}, User: ${_user}`);
-
     this._client = createClient({
       url: _url,
       max_open_connections: 10,
@@ -86,6 +87,9 @@ class ClickHouse {
     });
 
     await this._client.query({ query: 'SELECT 1' });
+
+    // start interval for batch insertion
+    setInterval(this.batchInsertCache.bind(this), interval);
   }
 
   /** Get the Clickhouse client (from @clickhouse/client) */
@@ -142,7 +146,7 @@ class ClickHouse {
    * @param {string} orgId - organization ID to add to each row
    * @param {string} deviceId - device ID to add to each row
    */
-  async insert(tableName, rows, orgId, deviceId) {
+  insert(tableName, rows, orgId, deviceId) {
     // assert that orgId and deviceId are provided
     if (!orgId || !deviceId) {
       throw new Error('Both orgId and deviceId must be provided for multi-tenant insert');
@@ -155,16 +159,13 @@ class ClickHouse {
       DeviceId: deviceId
     }));
 
-    return await this.client.insert({
-      table: tableName,
-      values: rowsWithIds,
-      format: 'JSONEachRow'
-    });
+    return this.addToCache(tableName, rowsWithIds);
   }
 
   /* Enable history recording. Ensure the mqtt_history table exists with the
    * correct schema, set dataCache, and subscribe to changes.
    * @param {object} options = {dataCache, tableName, ttlDays}
+   * @param {number} interval = ms interval between batch insertions
    */
   async enableHistory(options) {
     const { dataCache, tableName = 'mqtt_history' } = options;
@@ -241,7 +242,7 @@ class ClickHouse {
       // it matches any of the registered topics (this avoid duplicate triggers),
       // then store to ClickHouse with current timestamp.
       dataCache.subscribe((changes) => {
-        _.forEach(changes, async (value, topic) => {
+        _.forEach(changes, (value, topic) => {
 
           const matched =
             _.some(this.topics, (_true, selector) => topicMatch(selector, topic));
@@ -257,24 +258,40 @@ class ClickHouse {
             row.Payload = JSON.stringify(value);
           } // else: omit
 
-          try {
-            await this.client.insert({
-              table: this.mqttHistoryTable,
-              values: [row],
-              format: 'JSONEachRow'
-            });
-          } catch (error) {
-            console.error('Error inserting MQTT message into ClickHouse:', error.message);
-          }
+          // cache it for batch insertion
+          this.addToCache(this.mqttHistoryTable, [row]);
         })
       });
-
-
     }
 
     this.mqttHistoryTable = tableName;
   }
 
+  /** Add the given rows to the cache for batch-insertion to the given table */
+  addToCache(table, rows) {
+    this.rowCache[this.mqttHistoryTable] ||= [];
+    this.rowCache[this.mqttHistoryTable].push(...rows);
+  }
+
+  /** Function responsible fgor inserting all cached rows */
+  batchInsertCache() {
+    _.forEach(this.rowCache, (rows, table) => {
+      if (rows.length == 0) return;
+
+      try {
+        this.rowCache[table] = [];
+        this.client.insert({
+          table,
+          values: rows,
+          format: 'JSONEachRow'
+        });
+      } catch (error) {
+        console.error(`Error inserting ${rows.length} rows into ${table}`, error.message);
+      }
+    });
+  }
+
+
   /* Register an MQTT topic for storage in ClickHouse subscribes to the topic
   * and stores incoming messages JSON.stringify'd in a ClickHouse table.
   * Retrieve using `queryMQTTHistory`, or, when quering directly, e.g., from

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.4.3",
+  "version": "0.5.1",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {

package/test/clickhouse.test.js

@@ -48,7 +48,7 @@ describe('ClickHouse', function() {
   const dataCache = new DataCache({});
 
   before(async () => {
-    await clickhouse.init({ url: CLICKHOUSE_URL });
+    await clickhouse.init({ url: CLICKHOUSE_URL, interval: 100 });
     /* Register for `insert` events on ClickHouse client */
     emitter = interceptInserts();
 
@@ -63,7 +63,7 @@ describe('ClickHouse', function() {
     await clickhouse.enableHistory({
       dataCache,
       tableName: TABLE_NAME
-    });
+    }, 100);
 
     await clickhouse.registerMqttTopicForStorage(STANDARD_TOPIC_PATTERN);
   });
@@ -130,7 +130,6 @@ describe('ClickHouse', function() {
       await new Promise(resolve => setTimeout(resolve, 10));
       dataCache.update([org, 'device1', 'data'], null);
       await once(emitter, 'insert');
-      await once(emitter, 'insert');
 
       const rows = await queryRowsByOrg(org);