@transitive-sdk/clickhouse 0.3.2 → 0.3.4

package/index.js

@@ -1,5 +1,7 @@
 const _ = require('lodash');
+const waitPort = require('wait-port');
 const { createClient } = require('@clickhouse/client');
+
 const { topicToPath, topicMatch } = require('@transitive-sdk/datacache');
 
 // Default TTL in days for mqtt_history table
@@ -43,11 +45,21 @@ class ClickHouse {
   topics = {}; // list of topics registered for storage, as object for de-duplication
 
   /** Create the client, connecting to Clickhouse */
-  init({ url, dbName, user, password } = {}) {
+  async init({ url, dbName, user, password } = {}) {
+
     const _url = url || process.env.CLICKHOUSE_URL || 'http://clickhouse:8123';
     const _dbName = dbName || process.env.CLICKHOUSE_DB || 'default';
     const _user = user || process.env.CLICKHOUSE_USER || 'default';
 
+    const {hostname, port} = URL.parse(_url);
+    const interval = 200;
+    await waitPort({
+        host: hostname,
+        port: port || 80,
+        interval
+      }, 10000);
+    await new Promise(done => setTimeout(done, 200));
+
     // console.debug(`Creating ClickHouse client for URL: ${_url}, DB: ${_dbName}, User: ${_user}`);
 
     this._client = createClient({
@@ -72,6 +84,8 @@ class ClickHouse {
         // asterisk_include_materialized_columns: 1
       },
     });
+
+    await this._client.query({ query: 'SELECT 1' });
   }
 
   /** Get the Clickhouse client (from @clickhouse/client) */
@@ -83,6 +97,15 @@ class ClickHouse {
     return this._client;
   }
 
+  /* sets up default row level security policies in ClickHouse */
+  async ensureDefaultPermissions() {
+    const cmd = 'CREATE ROW POLICY IF NOT EXISTS';
+    for (let query of [
+      `${cmd} default_users ON default.* USING OrgId = splitByString('_', currentUser())[2] TO ALL`,
+      `${cmd} default_admin ON default.* USING 1 TO ${process.env.CLICKHOUSE_USER || 'default'}`
+    ]) await this.client.command({ query });
+  }
+
   /** Create a table if it does not already exist adding OrgId and DeviceId
    * columns to the schema for multi-tenancy support.
    * @param {string} tableName - name of the table to create
@@ -260,35 +283,26 @@ class ClickHouse {
   * @param {string} topic - MQTT topic to register
   */
   async registerMqttTopicForStorage(selector, ttlDays = DEFAULT_TTL_DAYS) {
-    this.topics[selector] = true;
-
-    // ---------------------------------------------------------------
-    // Set/update TTL for this capability and sub-topic
 
     const path = topicToPath(selector);
 
     if (path.length < 4) {
       // underspecified, don't set TTL
+      console.warn('Not registering topic as it is too short', selector);
       return;
     }
 
-    // list of TopicParts indices and selected value to use in WHERE statement
-    // const topicPartSelectors = [
-    //   [2, path[2]],
-    //   [3, path[3]]
-    // ];
+    this.topics[selector] = true;
 
-    // path.slice(5).forEach((value, i) => topicPartSelectors.push([i + 5, value]));
+    // ---------------------------------------------------------------
+    // Set/update TTL for this capability and sub-topic
 
-    // const where = topicPartSelectors
-    //     // filter out wildcards
-    //     .filter(([i, value]) => !['+','#'].includes(value[0]))
-    //     // map to WHERE conditions
-    //     .map(([i, value]) => `((TopicParts[${i + 1}]) = '${value}')`);
+    // Derive WHERE conditions for TTL expression from non-wildcards
     const where = path2where(path);
 
     if (where.length == 0) {
       // underspecified, don't set TTL
+      console.warn('Not setting TTL as topic is under specified', selector);
       return;
     }
 
@@ -307,17 +321,14 @@ class ClickHouse {
     const newTTLStatement =
       `toDateTime(Timestamp) + toIntervalDay(${ttlDays}) ${whereStatement}`;
 
-    let present = false;
-    // check if TTL statement already present on table definiton
-    ttls.forEach((ttl, i) => {
-      if (ttl.replace(/[()]/g, '').endsWith(whereStatement)) {
-        // condition already present, just replace it to update time
-        ttls[i] = newTTLStatement;
-        present = true;
-      }
-    });
+    const currentIndex =
+      ttls.findIndex(ttl => ttl.replace(/[()]/g, '').endsWith(whereStatement));
 
-    if (!present) {
+    if (currentIndex >= 0) {
+      // replace existing
+      ttls[currentIndex] = newTTLStatement;
+    } else {
+      // add new
       ttls.push(newTTLStatement);
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {
@@ -25,7 +25,8 @@
   },
   "dependencies": {
     "@clickhouse/client": "^1.12.1",
-    "@transitive-sdk/datacache": "^0.14.1"
+    "@transitive-sdk/datacache": "^0.14.1",
+    "wait-port": "^1.1.0"
   },
   "devDependencies": {
     "dotenv": "^17.2.3"

package/test/clickhouse.test.js

@@ -47,7 +47,7 @@ describe('ClickHouse', function() {
   const dataCache = new DataCache({});
 
   before(async () => {
-    clickhouse.init({ url: CLICKHOUSE_URL });
+    await clickhouse.init({ url: CLICKHOUSE_URL });
     /* Register for `insert` events on ClickHouse client */
     emitter = interceptInserts();