@budibase/server 2.0.39 → 2.0.40-alpha.0

package/src/integrations/mongodb.ts

@@ -6,13 +6,12 @@ import {
 } from "@budibase/types"
 import {
   MongoClient,
-  ObjectID,
-  FilterQuery,
-  UpdateQuery,
-  FindOneAndUpdateOption,
-  UpdateOneOptions,
-  UpdateManyOptions,
-  CommonOptions,
+  ObjectId,
+  Filter,
+  UpdateFilter,
+  FindOneAndUpdateOptions,
+  UpdateOptions,
+  OperationOptions,
 } from "mongodb"
 
 interface MongoDBConfig {
@@ -57,6 +56,232 @@ const SCHEMA: Integration = {
     delete: {
       type: QueryType.JSON,
     },
+    aggregate: {
+      type: QueryType.JSON,
+      steps: [
+        {
+          key: "$addFields",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$bucket",
+          template: `{
+  "groupBy": "",
+  "boundaries": [],
+  "default": "",
+  "output": {}
+}`,
+        },
+        {
+          key: "$bucketAuto",
+          template: `{
+  "groupBy": "",
+  "buckets": 1,
+  "output": {},
+  "granularity": "R5"
+}`,
+        },
+        {
+          key: "$changeStream",
+          template: `{
+  "allChangesForCluster": true,
+  "fullDocument": "",
+  "fullDocumentBeforeChange": "",
+  "resumeAfter": 1,
+  "showExpandedEvents": true,
+  "startAfter": {},
+  "startAtOperationTime": ""
+}`,
+        },
+        {
+          key: "$collStats",
+          template: `{
+  "latencyStats": { "histograms": true } },
+  "storageStats": { "scale": 1 } },
+  "count": {},
+  "queryExecStats": {}
+}`,
+        },
+        {
+          key: "$count",
+          template: ``,
+        },
+        {
+          key: "$densify",
+          template: `{
+  "field": "",
+  "partitionByFields": [],
+  "range": {
+    "step": 1,
+    "unit": 1,
+    "bounds": "full"
+  }
+}`,
+        },
+        {
+          key: "$documents",
+          template: `[]`,
+        },
+        {
+          key: "$facet",
+          template: `{\n\t\n}`,
+        },
+        {
+          key: "$fill",
+          template: `{
+  "partitionBy": "",
+  "partitionByFields": [],
+  "sortBy": {},
+  "output": {}
+}`,
+        },
+        {
+          key: "$geoNear",
+          template: `{
+  "near": { 
+    "type": "Point", 
+    "coordinates": [ 
+      -73.98142, 40.71782
+    ] 
+  },
+  "key": "location",
+  "distanceField": "dist.calculated",
+  "query": { "category": "Parks" }
+}`,
+        },
+        {
+          key: "$graphLookup",
+          template: `{
+  "from": "",
+  "startWith": "",
+  "connectFromField": "",
+  "connectToField": "",
+  "as": "",
+  "maxDepth": 1,
+  "depthField": "",
+  "restrictSearchWithMatch": {}
+}`,
+        },
+        {
+          key: "$group",
+          template: `{
+  "_id": ""
+}`,
+        },
+        {
+          key: "$indexStats",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$limit",
+          template: `1`,
+        },
+        {
+          key: "$listLocalSessions",
+          template: `{\n\t\n}`,
+        },
+        {
+          key: "$listSessions",
+          template: `{\n\t\n}`,
+        },
+        {
+          key: "$lookup",
+          template: `{
+  "from": "",
+  "localField": "",
+  "foreignField": "",
+  "as": ""
+}`,
+        },
+        {
+          key: "$match",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$merge",
+          template: `{
+  "into": {},
+  "on": "_id",
+  "whenMatched": "replace",
+  "whenNotMatched": "insert"
+}`,
+        },
+        {
+          key: "$out",
+          template: `{
+  "db": "",
+  "coll": ""
+}`,
+        },
+        {
+          key: "$planCacheStats",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$project",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$redact",
+          template: "",
+        },
+        {
+          key: "$replaceRoot",
+          template: `{ "newRoot": "" }`,
+        },
+        {
+          key: "$replaceWith",
+          template: ``,
+        },
+        {
+          key: "$sample",
+          template: `{ "size": 3 }`,
+        },
+        {
+          key: "$set",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$setWindowFields",
+          template: `{
+  "partitionBy": "",
+  "sortBy": {},
+  "output": {}
+}`,
+        },
+        {
+          key: "$skip",
+          template: `1`,
+        },
+        {
+          key: "$sort",
+          template: "{\n\t\n}",
+        },
+        {
+          key: "$sortByCount",
+          template: "",
+        },
+        {
+          key: "$unionWith",
+          template: `{
+  "coll": "",
+  "pipeline": []
+}`,
+        },
+        {
+          key: "$unset",
+          template: "",
+        },
+        {
+          key: "$unwind",
+          template: `{
+  "path": "",
+  "includeArrayIndex": "",
+  "preserveNullAndEmptyArrays": true
+}`,
+        },
+      ],
+    },
   },
   extra: {
     collection: {
@@ -64,8 +289,8 @@ const SCHEMA: Integration = {
       type: DatasourceFieldType.STRING,
       required: true,
     },
-    actionTypes: {
-      displayName: "Action Types",
+    actionType: {
+      displayName: "Query Type",
       type: DatasourceFieldType.LIST,
       required: true,
       data: {
@@ -73,6 +298,7 @@ const SCHEMA: Integration = {
         create: ["insertOne", "insertMany"],
         update: ["updateOne", "updateMany"],
         delete: ["deleteOne", "deleteMany"],
+        aggregate: ["json", "pipeline"],
       },
     },
   },
@@ -104,7 +330,7 @@ class MongoIntegration implements IntegrationBase {
         ) {
           const id = json[field].match(/(?<=objectid\(['"]).*(?=['"]\))/gi)?.[0]
           if (id) {
-            json[field] = ObjectID.createFromHexString(id)
+            json[field] = ObjectId.createFromHexString(id)
           }
         }
       }
@@ -168,7 +394,7 @@ class MongoIntegration implements IntegrationBase {
 
       // For mongodb we add an extra actionType to specify
       // which method we want to call on the collection
-      switch (query.extra.actionTypes) {
+      switch (query.extra.actionType) {
         case "insertOne": {
           return await collection.insertOne(json)
         }
@@ -177,7 +403,7 @@ class MongoIntegration implements IntegrationBase {
         }
         default: {
           throw new Error(
-            `actionType ${query.extra.actionTypes} does not exist on DB for create`
+            `actionType ${query.extra.actionType} does not exist on DB for create`
           )
         }
       }
@@ -196,7 +422,7 @@ class MongoIntegration implements IntegrationBase {
       const collection = db.collection(query.extra.collection)
       let json = this.createObjectIds(query.json)
 
-      switch (query.extra.actionTypes) {
+      switch (query.extra.actionType) {
         case "find": {
           return await collection.find(json).toArray()
         }
@@ -208,9 +434,9 @@ class MongoIntegration implements IntegrationBase {
             json = this.parseQueryParams(query.json, "update")
           }
           let findAndUpdateJson = this.createObjectIds(json) as {
-            filter: FilterQuery<any>
-            update: UpdateQuery<any>
-            options: FindOneAndUpdateOption<any>
+            filter: Filter<any>
+            update: UpdateFilter<any>
+            options: FindOneAndUpdateOptions
           }
           return await collection.findOneAndUpdate(
             findAndUpdateJson.filter,
@@ -226,7 +452,7 @@ class MongoIntegration implements IntegrationBase {
         }
         default: {
           throw new Error(
-            `actionType ${query.extra.actionTypes} does not exist on DB for read`
+            `actionType ${query.extra.actionType} does not exist on DB for read`
           )
         }
       }
@@ -248,29 +474,29 @@ class MongoIntegration implements IntegrationBase {
         queryJson = this.parseQueryParams(queryJson, "update")
       }
       let json = this.createObjectIds(queryJson) as {
-        filter: FilterQuery<any>
-        update: UpdateQuery<any>
+        filter: Filter<any>
+        update: UpdateFilter<any>
         options: object
       }
 
-      switch (query.extra.actionTypes) {
+      switch (query.extra.actionType) {
         case "updateOne": {
           return await collection.updateOne(
             json.filter,
             json.update,
-            json.options as UpdateOneOptions
+            json.options as UpdateOptions
           )
         }
         case "updateMany": {
           return await collection.updateMany(
             json.filter,
             json.update,
-            json.options as UpdateManyOptions
+            json.options as UpdateOptions
           )
         }
         default: {
           throw new Error(
-            `actionType ${query.extra.actionTypes} does not exist on DB for update`
+            `actionType ${query.extra.actionType} does not exist on DB for update`
           )
         }
       }
@@ -292,8 +518,8 @@ class MongoIntegration implements IntegrationBase {
         queryJson = this.parseQueryParams(queryJson, "delete")
       }
       let json = this.createObjectIds(queryJson) as {
-        filter: FilterQuery<any>
-        options: CommonOptions
+        filter: Filter<any>
+        options: OperationOptions
       }
       if (!json.options) {
         json = {
@@ -302,7 +528,7 @@ class MongoIntegration implements IntegrationBase {
         }
       }
 
-      switch (query.extra.actionTypes) {
+      switch (query.extra.actionType) {
         case "deleteOne": {
           return await collection.deleteOne(json.filter, json.options)
         }
@@ -311,7 +537,7 @@ class MongoIntegration implements IntegrationBase {
         }
         default: {
           throw new Error(
-            `actionType ${query.extra.actionTypes} does not exist on DB for delete`
+            `actionType ${query.extra.actionType} does not exist on DB for delete`
           )
         }
       }
@@ -322,6 +548,43 @@ class MongoIntegration implements IntegrationBase {
       await this.client.close()
     }
   }
+
+  async aggregate(query: {
+    json: object
+    steps: any[]
+    extra: { [key: string]: string }
+  }) {
+    try {
+      await this.connect()
+      const db = this.client.db(this.config.db)
+      const collection = db.collection(query.extra.collection)
+      let response = []
+      if (query.extra?.actionType === "pipeline") {
+        for await (const doc of collection.aggregate(
+          query.steps.map(({ key, value }) => {
+            let temp: any = {}
+            temp[key] = JSON.parse(value.value)
+            return this.createObjectIds(temp)
+          })
+        )) {
+          response.push(doc)
+        }
+      } else {
+        const stages: Array<any> = query.json as Array<any>
+        for await (const doc of collection.aggregate(
+          stages ? this.createObjectIds(stages) : []
+        )) {
+          response.push(doc)
+        }
+      }
+      return response
+    } catch (err) {
+      console.error("Error writing to mongodb", err)
+      throw err
+    } finally {
+      await this.client.close()
+    }
+  }
 }
 
 export default {

package/src/integrations/rest.ts

@@ -14,6 +14,7 @@ import {
   BearerAuthConfig,
 } from "../definitions/datasource"
 import { get } from "lodash"
+import * as https from "https"
 import qs from "querystring"
 const fetch = require("node-fetch")
 const { formatBytes } = require("../utilities")
@@ -76,11 +77,11 @@ const SCHEMA: Integration = {
       required: false,
       default: {},
     },
-    legacyHttpParser: {
-      display: "Legacy HTTP Support",
+    rejectUnauthorized: {
+      display: "Reject Unauthorized",
       type: DatasourceFieldType.BOOLEAN,
+      default: true,
       required: false,
-      default: false,
     },
   },
   query: {
@@ -218,8 +219,12 @@ class RestIntegration implements IntegrationBase {
       }
     }
 
-    // make sure the query string is fully encoded
-    const main = `${path}?${qs.encode(qs.decode(queryString))}`
+    if (queryString) {
+      // make sure the query string is fully encoded
+      queryString = "?" + qs.encode(qs.decode(queryString))
+    }
+    const main = `${path}${queryString}`
+
     let complete = main
     if (this.config.url && !main.startsWith("http")) {
       complete = !this.config.url ? main : `${this.config.url}/${main}`
@@ -381,6 +386,13 @@ class RestIntegration implements IntegrationBase {
       paginationValues
     )
 
+    if (this.config.rejectUnauthorized == false) {
+      input.agent = new https.Agent({
+        rejectUnauthorized: false,
+      })
+    }
+
+    // Deprecated by rejectUnauthorized
     if (this.config.legacyHttpParser) {
       // https://github.com/nodejs/node/issues/43798
       input.extraHttpOptions = { insecureHTTPParser: true }

package/src/integrations/tests/mongo.spec.ts

@@ -34,7 +34,7 @@ describe("MongoDB Integration", () => {
     await config.integration.create({
       index: indexName,
       json: body,
-      extra: { collection: "testCollection", actionTypes: "insertOne" },
+      extra: { collection: "testCollection", actionType: "insertOne" },
     })
     expect(config.integration.client.insertOne).toHaveBeenCalledWith(body)
   })
@@ -44,7 +44,7 @@ describe("MongoDB Integration", () => {
       json: {
         address: "test",
       },
-      extra: { collection: "testCollection", actionTypes: "find" },
+      extra: { collection: "testCollection", actionType: "find" },
     }
     const response = await config.integration.read(query)
     expect(config.integration.client.find).toHaveBeenCalledWith(query.json)
@@ -61,7 +61,7 @@ describe("MongoDB Integration", () => {
           opt: "option",
         },
       },
-      extra: { collection: "testCollection", actionTypes: "deleteOne" },
+      extra: { collection: "testCollection", actionType: "deleteOne" },
     }
     await config.integration.delete(query)
     expect(config.integration.client.deleteOne).toHaveBeenCalledWith(
@@ -83,7 +83,7 @@ describe("MongoDB Integration", () => {
           upsert: false,
         },
       },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
+      extra: { collection: "testCollection", actionType: "updateOne" },
     }
     await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalledWith(
@@ -97,7 +97,7 @@ describe("MongoDB Integration", () => {
     const restore = disableConsole()
 
     const query = {
-      extra: { collection: "testCollection", actionTypes: "deleteOne" },
+      extra: { collection: "testCollection", actionType: "deleteOne" },
     }
 
     let error = null
@@ -125,19 +125,19 @@ describe("MongoDB Integration", () => {
           upsert: false,
         },
       },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
+      extra: { collection: "testCollection", actionType: "updateOne" },
     }
     await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalled()
 
     const args = config.integration.client.updateOne.mock.calls[0]
     expect(args[0]).toEqual({
-      _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
-      name: mongo.ObjectID.createFromHexString("BBBB12345678ABCD12345678"),
+      _id: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
+      name: mongo.ObjectId.createFromHexString("BBBB12345678ABCD12345678"),
     })
     expect(args[1]).toEqual({
-      _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
-      name: mongo.ObjectID.createFromHexString("CCCC12345678ABCD12345678"),
+      _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"),
+      name: mongo.ObjectId.createFromHexString("CCCC12345678ABCD12345678"),
     })
     expect(args[2]).toEqual({
       upsert: false,
@@ -161,7 +161,7 @@ describe("MongoDB Integration", () => {
           upsert: true,
         },
       },
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
+      extra: { collection: "testCollection", actionType: "updateOne" },
     }
     await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalled()
@@ -169,12 +169,12 @@ describe("MongoDB Integration", () => {
     const args = config.integration.client.updateOne.mock.calls[0]
     expect(args[0]).toEqual({
       _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
       },
     })
     expect(args[1]).toEqual({
       $set: {
-        _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"),
+        _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"),
       },
     })
     expect(args[2]).toEqual({
@@ -200,7 +200,7 @@ describe("MongoDB Integration", () => {
           upsert: false,
         },
       },
-      extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" },
+      extra: { collection: "testCollection", actionType: "findOneAndUpdate" },
     }
     await config.integration.read(query)
     expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
@@ -208,7 +208,7 @@ describe("MongoDB Integration", () => {
     const args = config.integration.client.findOneAndUpdate.mock.calls[0]
     expect(args[0]).toEqual({
       _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
       },
     })
     expect(args[1]).toEqual({
@@ -245,7 +245,7 @@ describe("MongoDB Integration", () => {
         {
           "upsert": true
         }`,
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
+      extra: { collection: "testCollection", actionType: "updateOne" },
     }
     await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalled()
@@ -253,7 +253,7 @@ describe("MongoDB Integration", () => {
     const args = config.integration.client.updateOne.mock.calls[0]
     expect(args[0]).toEqual({
       _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
       },
     })
     expect(args[1]).toEqual({
@@ -300,7 +300,7 @@ describe("MongoDB Integration", () => {
           "upsert": true,
           "extra": "ad\\"{\\"d"
         }`,
-      extra: { collection: "testCollection", actionTypes: "updateOne" },
+      extra: { collection: "testCollection", actionType: "updateOne" },
     }
     await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalled()
@@ -308,7 +308,7 @@ describe("MongoDB Integration", () => {
     const args = config.integration.client.updateOne.mock.calls[0]
     expect(args[0]).toEqual({
       _id: {
-        $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"),
+        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
       },
     })
     expect(args[1]).toEqual({