@transitive-sdk/clickhouse 0.3.7 → 0.3.8

package/index.js

@@ -348,29 +348,17 @@ class ClickHouse {
       topicSelector,
       since = undefined,
       until = undefined,
-      orderBy = 'Timestamp ASC',
+      orderBy = 'Timestamp DESC',
       limit = 1000
     } = options;
 
     const path = topicToPath(topicSelector);
 
     // interpret wildcards
-    // const where = [];
-    // _.forEach(path, (value, i) => {
-    //   if (!['+','#'].includes(value[0])) {
-    //     // it's a constant, filter by it
-    //     where.push(`TopicParts[${i + 1}] = '${value}'`);
-    //     // Note that ClickHouse/SQL index starting at 1, not 0
-    //   }
-    // });
     const where = path2where(path);
-
     since && where.push(`Timestamp >= fromUnixTimestamp64Milli(${since.getTime()})`);
     until && where.push(`Timestamp <= fromUnixTimestamp64Milli(${until.getTime()})`);
-
-    const whereStatement = where.length > 0
-      ? `WHERE ${where.join(' AND ')}`
-      : '';
+    const whereStatement = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
 
     const result = await this.client.query({
       query: `SELECT Payload,TopicParts,Timestamp FROM default.${this.mqttHistoryTable} ${

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transitive-sdk/clickhouse",
-  "version": "0.3.7",
+  "version": "0.3.8",
   "description": "A tiny ClickHouse utility class for use in the Transitive framework.",
   "homepage": "https://transitiverobotics.com",
   "repository": {

package/test/clickhouse.test.js

@@ -33,6 +33,7 @@ const interceptInserts = () => {
 const queryRowsByOrg = async (org, options = {}) =>
   await clickhouse.queryMQTTHistory({
     topicSelector: `/${org}/+/+/+/+/+`,
+    orderBy: 'Timestamp ASC',
     ...options
   });
 
@@ -245,12 +246,14 @@ describe('ClickHouse', function() {
 
     it('queries with wild cards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/+` });
       assert(rows.length > 0);
     });
 
     it('queries with multiple selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/capdata/+/+` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepEqual(row.SubTopic, ['data']);
@@ -260,6 +263,7 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/data2` });
       assert.strictEqual(row.DeviceId, 'device1');
       assert.deepStrictEqual(row.Payload, { y: 2 });
@@ -267,12 +271,14 @@ describe('ClickHouse', function() {
 
     it('queries based on sub-topic selectors with wildcards', async () => {
       const [row] = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/+/sub2/+` });
       assert.deepStrictEqual(row.SubTopic[2], 'sub3.1');
     });
 
     it('queries based on multiple sub-topic selectors with wildcards', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/sub1/+/+` });
       assert.strictEqual(rows[0].SubTopic.length, 3);
       assert.strictEqual(rows[0].SubTopic[2], 'sub3.1');
@@ -281,6 +287,7 @@ describe('ClickHouse', function() {
 
     it('returns the history', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/data/+/+` });
       assert.deepStrictEqual(rows.length, 2);
       assert.deepStrictEqual(rows[0].Payload, {x: 1});
@@ -290,6 +297,7 @@ describe('ClickHouse', function() {
 
     it('handles null values', async () => {
       const rows = await clickhouse.queryMQTTHistory({
+        orderBy: 'Timestamp ASC',
         topicSelector: `/${org}/+/+/+/+/willBeNull` });
       assert.strictEqual(rows.at(-1).Payload, null);
     });
@@ -314,7 +322,8 @@ describe('ClickHouse', function() {
       for (let i = 0; i < ROWS; i++) {
        rows.push({
           Timestamp: new Date(now + i * GAP), // use current date to avoid immediate TTL cleanup
-          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope', `cap${i % 100}`, `1.${i % 100}.0`, 'data', i],
+          TopicParts: [`org${i % 50}`, `device${i % 1000}`, '@myscope',
+            `cap${i % 100}`, `1.${i % 100}.0`, `data_${i % 1000}`, i],
           Payload: { i },
        })
       }
@@ -345,7 +354,7 @@ describe('ClickHouse', function() {
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS);
-      assert(rows[0].Timestamp < rows[1].Timestamp);
+      assert(rows[0].Timestamp > rows[1].Timestamp);
       assertTimelimit(ROWS / 100);
     });
 
@@ -360,7 +369,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by DeviceId', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/device123/+/+/+/data`,
+        topicSelector: `/+/device123/+/+/+/+`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 1000);
@@ -369,16 +378,25 @@ describe('ClickHouse', function() {
 
     it('quickly filters by CapabilityName', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/cap34/+/data`,
+        topicSelector: `/+/+/+/cap34/+/+`,
         limit: 2 * ROWS,
       });
       assert.equal(rows.length, ROWS / 100);
       assertTimelimit(ROWS / 1000);
     });
 
+    it('quickly filters by SubTopic', async () => {
+      const rows = await clickhouse.queryMQTTHistory({
+        topicSelector: `/+/+/+/+/+/data_123`,
+        limit: 2 * ROWS,
+      });
+      assert.equal(rows.length, ROWS / 1000);
+      assertTimelimit(ROWS / 1000);
+    });
+
     it('quickly filters by time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/+`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });
@@ -388,7 +406,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by time: until', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/+/+/+/+/+/data`,
+        topicSelector: `/+/+/+/+/+/+`,
         until: new Date(now + 400 * GAP),
         limit: 2 * ROWS,
       });
@@ -398,7 +416,7 @@ describe('ClickHouse', function() {
 
     it('quickly filters by org and time: since', async () => {
       const rows = await clickhouse.queryMQTTHistory({
-        topicSelector: `/org23/+/+/+/+/data`,
+        topicSelector: `/org23/+/+/+/+/+`,
         since: new Date(now + (ROWS - 400) * GAP),
         limit: 2 * ROWS,
       });