@transitive-sdk/clickhouse 0.3.0 → 0.3.2

package/index.js

@@ -19,6 +19,21 @@ const MULTI_TENANT_SCHEMA = {
   ]
 };
 
+/** Given a Topic path (array), return an array of conditions to use in a
+ * WHERE clause. */
+const path2where = (path) => {
+  const where = [];
+  _.forEach(path, (value, i) => {
+    if (!['+','#'].includes(value[0])) {
+      // it's a constant, filter by it
+      where.push(`TopicParts[${i + 1}] = '${value}'`);
+      // Note that ClickHouse/SQL index starting at 1, not 0
+    }
+  });
+  return where;
+};
+
+
 /** Singleton ClickHouse client wrapper with multi-tenant table support */
 class ClickHouse {
 
@@ -258,18 +273,19 @@ class ClickHouse {
     }
 
     // list of TopicParts indices and selected value to use in WHERE statement
-    const topicPartSelectors = [
-      [2, path[2]],
-      [3, path[3]]
-    ];
+    // const topicPartSelectors = [
+    //   [2, path[2]],
+    //   [3, path[3]]
+    // ];
 
-    path.slice(5).forEach((value, i) => topicPartSelectors.push([i, value]));
+    // path.slice(5).forEach((value, i) => topicPartSelectors.push([i + 5, value]));
 
-    const where = topicPartSelectors
-        // filter out wildcards
-        .filter(([i, value]) => !['+','#'].includes(value[0]))
-        // map to WHERE conditions
-        .map(([i, value]) => `((TopicParts[${i}]) = '${value}')`);
+    // const where = topicPartSelectors
+    //     // filter out wildcards
+    //     .filter(([i, value]) => !['+','#'].includes(value[0]))
+    //     // map to WHERE conditions
+    //     .map(([i, value]) => `((TopicParts[${i + 1}]) = '${value}')`);
+    const where = path2where(path);
 
     if (where.length == 0) {
       // underspecified, don't set TTL
@@ -288,11 +304,13 @@ class ClickHouse {
     const ttls = matched ? matched[1].split(',').map(x => x.trim()) : [];
 
     const whereStatement = `WHERE ${where.join(' AND ')}`;
-    let present = false;
+    const newTTLStatement =
+      `toDateTime(Timestamp) + toIntervalDay(${ttlDays}) ${whereStatement}`;
 
+    let present = false;
     // check if TTL statement already present on table definiton
     ttls.forEach((ttl, i) => {
-      if (ttl.endsWith(whereStatement)) {
+      if (ttl.replace(/[()]/g, '').endsWith(whereStatement)) {
         // condition already present, just replace it to update time
         ttls[i] = newTTLStatement;
         present = true;
@@ -300,7 +318,7 @@ class ClickHouse {
     });
 
     if (!present) {
-      ttls.push(`toDateTime(Timestamp) + toIntervalDay(${ttlDays}) ${whereStatement}`);
+      ttls.push(newTTLStatement);
     }
 
     await this.client.command({
@@ -327,14 +345,15 @@ class ClickHouse {
     const path = topicToPath(topicSelector);
 
     // interpret wildcards
-    const where = [];
-    _.forEach(path, (value, i) => {
-      if (!['+','#'].includes(value[0])) {
-        // it's a constant, filter by it
-        where.push(`TopicParts[${i + 1}] = '${value}'`);
-        // Note that ClickHouse/SQL index starting at 1, not 0
-      }
-    });
+    // const where = [];
+    // _.forEach(path, (value, i) => {
+    //   if (!['+','#'].includes(value[0])) {
+    //     // it's a constant, filter by it
+    //     where.push(`TopicParts[${i + 1}] = '${value}'`);
+    //     // Note that ClickHouse/SQL index starting at 1, not 0
+    //   }
+    // });
+    const where = path2where(path);
 
     since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
     until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {

package/test/clickhouse.test.js

@@ -61,8 +61,7 @@ describe('ClickHouse', function() {
 
     await clickhouse.enableHistory({
       dataCache,
-      tableName: TABLE_NAME,
-      ttlDays: 31
+      tableName: TABLE_NAME
     });
 
     await clickhouse.registerMqttTopicForStorage(STANDARD_TOPIC_PATTERN);
@@ -194,6 +193,10 @@ describe('ClickHouse', function() {
       assert.equal(rows.length, 1);
     });
 
+    it('updates TTL without crashing', async () => {
+      await clickhouse.registerMqttTopicForStorage('/+/+/myscope/+/+/#', 13);
+      await clickhouse.registerMqttTopicForStorage('/+/+/myscope/+/+/#', 15);
+    });
   });