@budibase/server 2.4.41 → 2.4.42-alpha.1

package/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -1,48 +1,48 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
-      doc[\\"myField\\"] === undefined ||
-      doc[\\"myField\\"] === null ||
-      doc[\\"myField\\"] === \\"\\" ||
-      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
-  )) && (doc[\\"age\\"] > 17)) {
-        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
+      doc["myField"] === undefined ||
+      doc["myField"] === null ||
+      doc["myField"] === "" ||
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
+  )) && (doc["age"] > 17)) {
+        emit(doc["_id"], doc["myField"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": "stats",
     "field": "myField",
-    "filters": Array [
-      Object {
+    "filters": [
+      {
         "condition": "MT",
         "key": "age",
         "value": 17,
       },
     ],
     "groupBy": undefined,
-    "schema": Object {
-      "avg": Object {
+    "schema": {
+      "avg": {
         "type": "number",
       },
-      "count": Object {
+      "count": {
         "type": "number",
       },
-      "field": Object {
+      "field": {
         "type": "string",
       },
-      "max": Object {
+      "max": {
         "type": "number",
       },
-      "min": Object {
+      "min": {
         "type": "number",
       },
-      "sum": Object {
+      "sum": {
         "type": "number",
       },
-      "sumsqr": Object {
+      "sumsqr": {
         "type": "number",
       },
     },
@@ -53,42 +53,42 @@ Object {
 `;
 
 exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
-      doc[\\"myField\\"] === undefined ||
-      doc[\\"myField\\"] === null ||
-      doc[\\"myField\\"] === \\"\\" ||
-      (Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
+      if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
+      doc["myField"] === undefined ||
+      doc["myField"] === null ||
+      doc["myField"] === "" ||
+      (Array.isArray(doc["myField"]) && doc["myField"].length === 0)
   )) ) {
-        emit(doc[\\"_id\\"], doc[\\"myField\\"]);
+        emit(doc["_id"], doc["myField"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": "stats",
     "field": "myField",
-    "filters": Array [],
+    "filters": [],
     "groupBy": undefined,
-    "schema": Object {
-      "avg": Object {
+    "schema": {
+      "avg": {
         "type": "number",
       },
-      "count": Object {
+      "count": {
         "type": "number",
       },
-      "field": Object {
+      "field": {
         "type": "string",
       },
-      "max": Object {
+      "max": {
         "type": "number",
       },
-      "min": Object {
+      "min": {
         "type": "number",
       },
-      "sum": Object {
+      "sum": {
         "type": "number",
       },
-      "sumsqr": Object {
+      "sumsqr": {
         "type": "number",
       },
     },
@@ -99,22 +99,22 @@ Object {
 `;
 
 exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && (doc[\\"Name\\"] === \\"Test\\" || doc[\\"Yes\\"] > \\"Value\\")) {
-        emit(doc[\\"_id\\"], doc[\\"undefined\\"]);
+      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
+        emit(doc["_id"], doc["undefined"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": undefined,
     "field": undefined,
-    "filters": Array [
-      Object {
+    "filters": [
+      {
         "condition": "EQUALS",
         "key": "Name",
         "value": "Test",
       },
-      Object {
+      {
         "condition": "MT",
         "conjunction": "OR",
         "key": "Yes",
@@ -129,16 +129,16 @@ Object {
 `;
 
 exports[`viewBuilder Group By creates a view emitting the group by field 1`] = `
-Object {
+{
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
-        emit(doc[\\"age\\"], doc[\\"score\\"]);
+      if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) {
+        emit(doc["age"], doc["score"]);
       }
     }",
-  "meta": Object {
+  "meta": {
     "calculation": undefined,
     "field": "score",
-    "filters": Array [],
+    "filters": [],
     "groupBy": "age",
     "schema": null,
     "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",

package/src/api/routes/public/index.ts

@@ -1,4 +1,5 @@
 import appEndpoints from "./applications"
+import metricEndpoints from "./metrics"
 import queryEndpoints from "./queries"
 import tableEndpoints from "./tables"
 import rowEndpoints from "./rows"
@@ -12,7 +13,7 @@ import env from "../../../environment"
 // below imports don't have declaration files
 const Router = require("@koa/router")
 const { RateLimit, Stores } = require("koa2-ratelimit")
-import { redis, permissions } from "@budibase/backend-core"
+import { middleware, redis, permissions } from "@budibase/backend-core"
 const { PermissionType, PermissionLevel } = permissions
 
 const PREFIX = "/api/public/v1"
@@ -91,6 +92,13 @@ function addToRouter(endpoints: any) {
   }
 }
 
+function applyAdminRoutes(endpoints: any) {
+  addMiddleware(endpoints.read, middleware.builderOrAdmin)
+  addMiddleware(endpoints.write, middleware.builderOrAdmin)
+  addToRouter(endpoints.read)
+  addToRouter(endpoints.write)
+}
+
 function applyRoutes(
   endpoints: any,
   permType: string,
@@ -119,6 +127,7 @@ function applyRoutes(
   addToRouter(endpoints.write)
 }
 
+applyAdminRoutes(metricEndpoints)
 applyRoutes(appEndpoints, PermissionType.APP, "appId")
 applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId")
 applyRoutes(userEndpoints, PermissionType.USER, "userId")

package/src/api/routes/public/metrics.ts

@@ -0,0 +1,28 @@
+import controller from "../../controllers/public/metrics"
+import Endpoint from "./utils/Endpoint"
+
+const read = []
+
+/**
+ * @openapi
+ * /metrics:
+ *   get:
+ *     operationId: metricsGet
+ *     summary: Retrieve Budibase tenant metrics
+ *     description: Output metrics in OpenMetrics format compatible with Prometheus
+ *     tags:
+ *       - metrics
+ *     responses:
+ *       200:
+ *         description: Returns tenant metrics.
+ *         content:
+ *           text/plain:
+ *             schema:
+ *               type: string
+ *             examples:
+ *               metrics:
+ *                 $ref: '#/components/examples/metrics'
+ */
+read.push(new Endpoint("get", "/metrics", controller.fetch))
+
+export default { read }

package/src/api/routes/public/tests/metrics.spec.js

@@ -0,0 +1,34 @@
+const setup = require("../../tests/utilities")
+
+jest.setTimeout(30000)
+
+describe("/metrics", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  // For some reason this cannot be a beforeAll or the test "should be able to update the user" fail
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  describe("get", () => {
+    it("returns a list of metrics", async () => {
+      const res = await request
+        .get(`/api/public/v1/metrics`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /text\/plain/)
+        .expect(200)
+      expect(res.text).toContain("budibase_tenant_user_count")
+    })
+
+    it("endpoint should not be publicly exposed", async () => {
+      await request
+      .get(`/api/public/v1/metrics`)
+      .set(config.publicHeaders())
+      .expect(403)
+    })
+  })
+
+})

package/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap

@@ -1,21 +1,21 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`/datasources fetch returns all the datasources from the server 1`] = `
-Array [
-  Object {
-    "config": Object {},
-    "entities": Array [
-      Object {
+[
+  {
+    "config": {},
+    "entities": [
+      {
         "_id": "ta_users",
         "_rev": "1-2375e1bc58aeec664dc1b1f04ad43e44",
         "createdAt": "2020-01-01T00:00:00.000Z",
         "name": "Users",
         "primaryDisplay": "email",
-        "schema": Object {
-          "email": Object {
-            "constraints": Object {
+        "schema": {
+          "email": {
+            "constraints": {
               "email": true,
-              "length": Object {
+              "length": {
                 "maximum": "",
               },
               "presence": true,
@@ -25,8 +25,8 @@ Array [
             "name": "email",
             "type": "string",
           },
-          "firstName": Object {
-            "constraints": Object {
+          "firstName": {
+            "constraints": {
               "presence": false,
               "type": "string",
             },
@@ -34,8 +34,8 @@ Array [
             "name": "firstName",
             "type": "string",
           },
-          "lastName": Object {
-            "constraints": Object {
+          "lastName": {
+            "constraints": {
               "presence": false,
               "type": "string",
             },
@@ -43,9 +43,9 @@ Array [
             "name": "lastName",
             "type": "string",
           },
-          "roleId": Object {
-            "constraints": Object {
-              "inclusion": Array [
+          "roleId": {
+            "constraints": {
+              "inclusion": [
                 "ADMIN",
                 "POWER",
                 "BASIC",
@@ -58,9 +58,9 @@ Array [
             "name": "roleId",
             "type": "options",
           },
-          "status": Object {
-            "constraints": Object {
-              "inclusion": Array [
+          "status": {
+            "constraints": {
+              "inclusion": [
                 "active",
                 "inactive",
               ],
@@ -74,15 +74,15 @@ Array [
         },
         "type": "table",
         "updatedAt": "2020-01-01T00:00:00.000Z",
-        "views": Object {},
+        "views": {},
       },
     ],
     "name": "Budibase DB",
     "source": "BUDIBASE",
     "type": "budibase",
   },
-  Object {
-    "config": Object {},
+  {
+    "config": {},
     "createdAt": "2020-01-01T00:00:00.000Z",
     "name": "Test",
     "source": "POSTGRES",

package/src/api/routes/tests/__snapshots__/view.spec.js.snap

@@ -1,8 +1,8 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`/views query returns data for the created view 1`] = `
-Array [
-  Object {
+[
+  {
     "avg": 2333.3333333333335,
     "count": 3,
     "group": null,
@@ -15,8 +15,8 @@ Array [
 `;
 
 exports[`/views query returns data for the created view using a group by 1`] = `
-Array [
-  Object {
+[
+  {
     "avg": 1500,
     "count": 2,
     "group": "One",
@@ -25,7 +25,7 @@ Array [
     "sum": 3000,
     "sumsqr": 5000000,
   },
-  Object {
+  {
     "avg": 4000,
     "count": 1,
     "group": "Two",

package/src/api/routes/tests/appSync.spec.ts

@@ -0,0 +1,31 @@
+import * as setup from "./utilities"
+import { roles, db as dbCore } from "@budibase/backend-core"
+
+describe("/api/applications/:appId/sync", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeAll(async () => {
+    app = await config.init()
+    // create some users which we will use throughout the tests
+    await config.createUser({
+      email: "sync1@test.com",
+      roles: {
+        [app._id!]: roles.BUILTIN_ROLE_IDS.BASIC,
+      },
+    })
+  })
+
+  async function getUserMetadata() {
+    const { rows } = await config.searchRows(dbCore.InternalTable.USER_METADATA)
+    return rows
+  }
+
+  it("make sure that user metadata is correctly sync'd", async () => {
+    const rows = await getUserMetadata()
+    expect(rows.length).toBe(1)
+  })
+})

package/src/api/routes/tests/internalSearch.spec.js

@@ -1,6 +1,7 @@
+const fetch = require("node-fetch")
+fetch.mockSearch()
 const search = require("../../controllers/row/internalSearch")
 // this will be mocked out for _search endpoint
-const fetch = require("node-fetch")
 const PARAMS = {
   tableId: "ta_12345679abcdef",
   version: "1",
@@ -20,7 +21,7 @@ function checkLucene(resp, expected, params = PARAMS) {
     expect(json.bookmark).toBe(PARAMS.bookmark)
   }
   expect(json.include_docs).toBe(true)
-  expect(json.q).toBe(`(${expected}) AND tableId:"${params.tableId}"`)
+  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
   expect(json.limit).toBe(params.limit || 50)
 }
 
@@ -59,7 +60,7 @@ describe("internal search", () => {
         "column": "1",
       }
     }, PARAMS)
-    checkLucene(response, `column:"2" OR !column:"1"`)
+    checkLucene(response, `(column:"2" OR !column:"1")`)
   })
 
   it("test AND query", async () => {
@@ -71,7 +72,7 @@ describe("internal search", () => {
         "column": "1",
       }
     }, PARAMS)
-    checkLucene(response, `*:* AND column:"2" AND !column:"1"`)
+    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
   })
 
   it("test pagination query", async () => {
@@ -132,7 +133,7 @@ describe("internal search", () => {
         "colArr": [1, 2, 3],
       },
     }, PARAMS)
-    checkLucene(response, `*:* AND column:a AND colArr:(1 AND 2 AND 3)`, PARAMS)
+    checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
   })
 
   it("test multiple of same column", async () => {
@@ -144,7 +145,7 @@ describe("internal search", () => {
         "3:column": "c",
       },
     }, PARAMS)
-    checkLucene(response, `column:"a" OR column:"b" OR column:"c"`, PARAMS)
+    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
   })
 
   it("check a weird case for lucene building", async () => {
@@ -191,6 +192,6 @@ describe("internal search", () => {
       expect(json.bookmark).toBe(PARAMS.bookmark)
     }
     expect(json.include_docs).toBe(true)
-    expect(json.q).toBe(`(*:* AND column:"1") AND tableId:${PARAMS.tableId}`)
+    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
   })
 })

package/src/app.ts

@@ -27,7 +27,7 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { events, logging, middleware } from "@budibase/backend-core"
+import { events, logging, middleware, timers } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
@@ -84,6 +84,7 @@ server.on("close", async () => {
   }
   shuttingDown = true
   console.log("Server Closed")
+  timers.cleanup()
   await automations.shutdown()
   await redis.shutdown()
   events.shutdown()

package/src/automations/automationUtils.ts

@@ -23,7 +23,7 @@ import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
  * @returns {object} The inputs object which has had all the various types supported by this function converted to their
  * primitive types.
  */
-export function cleanInputValues(inputs: Record<string, any>, schema: any) {
+export function cleanInputValues(inputs: Record<string, any>, schema?: any) {
   if (schema == null) {
     return inputs
   }

package/src/automations/tests/automation.spec.ts

@@ -0,0 +1,99 @@
+jest.mock("../../threads/automation")
+jest.mock("../../utilities/redis", () => ({
+  init: jest.fn(),
+  checkTestFlag: () => {
+    return false
+  },
+}))
+
+jest.spyOn(global.console, "error")
+
+import "../../environment"
+import * as automation from "../index"
+import * as thread from "../../threads/automation"
+import * as triggers from "../triggers"
+import { basicAutomation } from "../../tests/utilities/structures"
+import { wait } from "../../utilities"
+import { makePartial } from "../../tests/utilities"
+import { cleanInputValues } from "../automationUtils"
+import * as setup from "./utilities"
+import { Automation } from "@budibase/types"
+
+describe("Run through some parts of the automations system", () => {
+  let config = setup.getConfig()
+
+  beforeAll(async () => {
+    await automation.init()
+    await config.init()
+  })
+
+  afterAll(async () => {
+    await automation.shutdown()
+    setup.afterAll()
+  })
+
+  it("should be able to init in builder", async () => {
+    const automation: Automation = {
+      ...basicAutomation(),
+      appId: config.appId,
+    }
+    const fields: any = { a: 1, appId: config.appId }
+    await triggers.externalTrigger(automation, fields)
+    await wait(100)
+    expect(thread.execute).toHaveBeenCalled()
+  })
+
+  it("should check coercion", async () => {
+    const table = await config.createTable()
+    const automation: any = basicAutomation()
+    automation.definition.trigger.inputs.tableId = table._id
+    automation.definition.trigger.stepId = "APP"
+    automation.definition.trigger.inputs.fields = { a: "number" }
+    const fields: any = {
+      appId: config.getAppId(),
+      fields: {
+        a: "1",
+      },
+    }
+    await triggers.externalTrigger(automation, fields)
+    await wait(100)
+    expect(thread.execute).toHaveBeenCalledWith(
+      makePartial({
+        data: {
+          event: {
+            fields: {
+              a: 1,
+            },
+          },
+        },
+      }),
+      expect.any(Function)
+    )
+  })
+
+  it("should be able to clean inputs with the utilities", () => {
+    // can't clean without a schema
+    let output = cleanInputValues({ a: "1" })
+    expect(output.a).toBe("1")
+    output = cleanInputValues(
+      { a: "1", b: "true", c: "false", d: 1, e: "help" },
+      {
+        properties: {
+          a: {
+            type: "number",
+          },
+          b: {
+            type: "boolean",
+          },
+          c: {
+            type: "boolean",
+          },
+        },
+      }
+    )
+    expect(output.a).toBe(1)
+    expect(output.b).toBe(true)
+    expect(output.c).toBe(false)
+    expect(output.d).toBe(1)
+  })
+})

package/src/definitions/openapi.ts

@@ -22,6 +22,10 @@ export interface paths {
     /** Based on application properties (currently only name) search for applications. */
     post: operations["appSearch"];
   };
+  "/metrics": {
+    /** Output metrics in OpenMetrics format compatible with Prometheus */
+    get: operations["metricsGet"];
+  };
   "/queries/{queryId}": {
     /** Queries which have been created within a Budibase app can be executed using this, */
     post: operations["queryExecute"];
@@ -844,6 +848,17 @@ export interface operations {
       };
     };
   };
+  /** Output metrics in OpenMetrics format compatible with Prometheus */
+  metricsGet: {
+    responses: {
+      /** Returns tenant metrics. */
+      200: {
+        content: {
+          "text/plain": string;
+        };
+      };
+    };
+  };
   /** Queries which have been created within a Budibase app can be executed using this, */
   queryExecute: {
     parameters: {