@budibase/server 1.0.219 → 1.0.220-alpha.2

package/src/api/controllers/row/internalSearch.js

@@ -10,6 +10,7 @@ const { getAppId } = require("@budibase/backend-core/context")
 class QueryBuilder {
   constructor(base) {
     this.query = {
+      allOr: false,
       string: {},
       fuzzy: {},
       range: {},
@@ -17,6 +18,7 @@ class QueryBuilder {
       notEqual: {},
       empty: {},
       notEmpty: {},
+      oneOf: {},
       ...base,
     }
     this.limit = 50
@@ -112,6 +114,11 @@ class QueryBuilder {
     return this
   }
 
+  addOneOf(key, value) {
+    this.query.oneOf[key] = value
+    return this
+  }
+
   /**
    * Preprocesses a value before going into a lucene search.
    * Transforms strings to lowercase and wraps strings and bools in quotes.
@@ -140,8 +147,22 @@ class QueryBuilder {
 
   buildSearchQuery() {
     const builder = this
-    let query = "*:*"
+    let allOr = this.query && this.query.allOr
+    let query = allOr ? "" : "*:*"
     const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
+    let tableId
+    if (this.query.equal.tableId) {
+      tableId = this.query.equal.tableId
+      delete this.query.equal.tableId
+    }
+
+    const equal = (key, value) => {
+      // 0 evaluates to false, which means we would return all rows if we don't check it
+      if (!value && value !== 0) {
+        return null
+      }
+      return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
+    }
 
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
@@ -152,7 +173,10 @@ class QueryBuilder {
         if (expression == null) {
           continue
         }
-        query += ` AND ${expression}`
+        if (query.length > 0) {
+          query += ` ${allOr ? "OR" : "AND"} `
+        }
+        query += expression
       }
     }
 
@@ -198,13 +222,7 @@ class QueryBuilder {
       })
     }
     if (this.query.equal) {
-      build(this.query.equal, (key, value) => {
-        // 0 evaluates to false, which means we would return all rows if we don't check it
-        if (!value && value !== 0) {
-          return null
-        }
-        return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
-      })
+      build(this.query.equal, equal)
     }
     if (this.query.notEqual) {
       build(this.query.notEqual, (key, value) => {
@@ -220,6 +238,34 @@ class QueryBuilder {
     if (this.query.notEmpty) {
       build(this.query.notEmpty, key => `${key}:["" TO *]`)
     }
+    if (this.query.oneOf) {
+      build(this.query.oneOf, (key, value) => {
+        if (!Array.isArray(value)) {
+          if (typeof value === "string") {
+            value = value.split(",")
+          } else {
+            return ""
+          }
+        }
+        let orStatement = `${builder.preprocess(
+          value[0],
+          allPreProcessingOpts
+        )}`
+        for (let i = 1; i < value.length; i++) {
+          orStatement += ` OR ${builder.preprocess(
+            value[i],
+            allPreProcessingOpts
+          )}`
+        }
+        return `${key}:(${orStatement})`
+      })
+    }
+    // make sure table ID is always added as an AND
+    if (tableId) {
+      query = `(${query})`
+      allOr = false
+      build({ tableId }, equal)
+    }
     return query
   }
 

package/src/api/controllers/row/utils.js

@@ -64,6 +64,9 @@ exports.validate = async ({ tableId, row, table }) => {
     // Validate.js doesn't seem to handle array
     if (type === FieldTypes.ARRAY && row[fieldName]) {
       if (row[fieldName].length) {
+        if (!Array.isArray(row[fieldName])) {
+          row[fieldName] = row[fieldName].split(",")
+        }
         row[fieldName].map(val => {
           if (
             !constraints.inclusion.includes(val) &&

package/src/api/controllers/static/index.js

@@ -73,12 +73,7 @@ exports.toggleBetaUiFeature = async function (ctx) {
 }
 
 exports.serveBuilder = async function (ctx) {
-  // Temporary: New Design UI
-  const designUiCookie = ctx.cookies.get("beta:design_ui")
-  // TODO: get this from the tmp Dir that we downloaded from MinIO
-  const uiPath = designUiCookie ? "new_design_ui" : "builder"
-
-  let builderPath = resolve(TOP_LEVEL_PATH, uiPath)
+  const builderPath = resolve(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
   if (!ctx.file.includes("assets/")) {
     await events.serve.servedBuilder()

package/src/api/index.js

@@ -72,7 +72,10 @@ router.use(async (ctx, next) => {
       error,
     }
     ctx.log.error(err)
-    console.trace(err)
+    // unauthorised errors don't provide a useful trace
+    if (!env.isTest()) {
+      console.trace(err)
+    }
   }
 })
 

package/src/api/routes/automation.js

@@ -51,6 +51,16 @@ router
     automationValidator(false),
     controller.create
   )
+  .post(
+    "/api/automations/logs/search",
+    authorized(BUILDER),
+    controller.logSearch
+  )
+  .delete(
+    "/api/automations/logs",
+    authorized(BUILDER),
+    controller.clearLogError
+  )
   .delete(
     "/api/automations/:id/:rev",
     paramResource("id"),

package/src/api/routes/dev.js

@@ -22,6 +22,5 @@ router
   .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
   .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
   .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
-  .get("/api/beta/access", controller.checkBetaAccess)
 
 module.exports = router

package/src/api/routes/public/rows.ts

@@ -150,6 +150,11 @@ read.push(new Endpoint("get", "/tables/:tableId/rows/:rowId", controller.read))
  *               query:
  *                 type: object
  *                 properties:
+ *                   allOr:
+ *                     type: boolean
+ *                     description: Specifies that a row should be returned if it satisfies
+ *                       any of the specified options, rather than requiring it to fulfill all
+ *                       the search parameters. This defaults to false, meaning AND logic will be used.
  *                   string:
  *                     type: object
  *                     example:
@@ -167,9 +172,9 @@ read.push(new Endpoint("get", "/tables/:tableId/rows/:rowId", controller.read))
  *                   range:
  *                     type: object
  *                     description: Searches within a range, the format of this must be
- *                       columnName -> [low, high].
+ *                       in the format of an object with a "low" and "high" property.
  *                     example:
- *                       columnName1: [10, 20]
+ *                       columnName1: { low: 10, high: 20 }
  *                   equal:
  *                     type: object
  *                     description: Searches for rows that have a column value that is

package/src/api/routes/tests/query.spec.js

@@ -218,7 +218,6 @@ describe("/queries", () => {
       expect(res.body.schemaFields).toEqual(["a", "b"])
       expect(res.body.rows.length).toEqual(1)
       expect(events.query.previewed).toBeCalledTimes(1)
-      datasource.config = { schema: "public" }
       expect(events.query.previewed).toBeCalledWith(datasource, query)
     })
 
@@ -346,4 +345,170 @@ describe("/queries", () => {
       expect(contents).toBe(null)
     })
   })
+
+  describe("Current User Request Mapping", () => {
+    
+    async function previewGet(datasource, fields, params) {
+      return config.previewQuery(request, config, datasource, fields, params)
+    }
+
+    async function previewPost(datasource, fields, params) {
+      return config.previewQuery(request, config, datasource, fields, params, "create")
+    }
+
+    it("should parse global and query level header mappings", async () => {
+      const userDetails = config.getUserDetails()
+
+      const datasource = await config.restDatasource({
+        defaultHeaders: {
+          "test": "headerVal",
+          "emailHdr": "{{[user].[email]}}"
+        }
+      })
+      const res = await previewGet(datasource, {
+        path: "www.google.com",
+        queryString: "email={{[user].[email]}}",
+        headers: {
+          queryHdr : "{{[user].[firstName]}}",
+          secondHdr : "1234"
+        }
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      expect(parsedRequest.opts.headers).toEqual({
+        "test": "headerVal",
+        "emailHdr": userDetails.email,
+        "queryHdr": userDetails.firstName,
+        "secondHdr" : "1234"
+      })
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email)
+    })
+
+    it("should bind the current user to query parameters", async () => {
+      const userDetails = config.getUserDetails()
+  
+      const datasource = await config.restDatasource()
+  
+      const res = await previewGet(datasource, {
+        path: "www.google.com",
+        queryString: "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
+      }, {
+        "myEmail" : "{{[user].[email]}}",
+        "myName" : "{{[user].[firstName]}}",
+        "testParam" : "1234"
+      })
+  
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email + 
+        "&testName=" + userDetails.firstName + "&testParam=1234")
+    })
+
+    it("should bind the current user the request body - plain text", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
+        bodyType: "text"
+      }, {
+        "testParam" : "1234"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      expect(parsedRequest.opts.body).toEqual(`This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - json", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "json"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+   
+    it("should bind the current user the request body - xml", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "<note> <email>{{[user].[email]}}</email> <code>{{testParam}}</code> " + 
+          "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
+        bodyType: "xml"
+      }, {
+        "testParam" : "1234",
+        "userId" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
+        
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - form-data", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "form"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+
+      const emailData = parsedRequest.opts.body._streams[1]
+      expect(emailData).toEqual(userDetails.email)
+
+      const queryCodeData = parsedRequest.opts.body._streams[4]
+      expect(queryCodeData).toEqual("1234")
+
+      const userRef = parsedRequest.opts.body._streams[7]
+      expect(userRef).toEqual(userDetails.firstName)
+
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - encoded", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "encoded"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      
+      expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
+      expect(parsedRequest.opts.body.queryCode).toEqual("1234")
+      expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName)
+    })
+
+  });
 })

package/src/api/routes/utils/validators.js

@@ -58,6 +58,7 @@ function filterObject() {
     oneOf: Joi.object().optional(),
     contains: Joi.object().optional(),
     notContains: Joi.object().optional(),
+    allOr: Joi.boolean().optional(),
   }).unknown(true)
 }
 

package/src/app.ts

@@ -18,7 +18,9 @@ const { logAlert } = require("@budibase/backend-core/logging")
 const { Thread } = require("./threads")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
-import { events, installation } from "@budibase/backend-core"
+import { events, installation, tenancy } from "@budibase/backend-core"
+import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { tenantSucceeded } from "@budibase/backend-core/dist/src/events/publishers/backfill"
 
 const app = new Koa()
 
@@ -110,6 +112,33 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
+  // check and create admin user if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.BB_ADMIN_USER_EMAIL &&
+    env.BB_ADMIN_USER_PASSWORD
+  ) {
+    const checklist = await getChecklist()
+    if (!checklist?.adminUser?.checked) {
+      try {
+        const tenantId = tenancy.getTenantId()
+        await createAdminUser(
+          env.BB_ADMIN_USER_EMAIL,
+          env.BB_ADMIN_USER_PASSWORD,
+          tenantId
+        )
+        console.log(
+          "Admin account automatically created for",
+          env.BB_ADMIN_USER_EMAIL
+        )
+      } catch (e) {
+        logAlert("Error creating initial admin user. Exiting.", e)
+        shutdown()
+      }
+    }
+  }
+
   // check for version updates
   await installation.checkInstallVersion()
 

package/src/automations/automationUtils.js

@@ -1,5 +1,10 @@
 const { getTable } = require("../api/controllers/table/utils")
-const { findHBSBlocks } = require("@budibase/string-templates")
+const {
+  findHBSBlocks,
+  decodeJSBinding,
+  isJSBinding,
+  encodeJSBinding,
+} = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -77,11 +82,21 @@ exports.getError = err => {
 }
 
 exports.substituteLoopStep = (hbsString, substitute) => {
-  let blocks = findHBSBlocks(hbsString)
+  let blocks = []
+  let checkForJS = isJSBinding(hbsString)
+  if (checkForJS) {
+    hbsString = decodeJSBinding(hbsString)
+    blocks.push(hbsString)
+  } else {
+    blocks = findHBSBlocks(hbsString)
+  }
   for (let block of blocks) {
-    let oldBlock = block
     block = block.replace(/loop/, substitute)
-    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
+    if (checkForJS) {
+      hbsString = encodeJSBinding(block)
+    } else {
+      hbsString = block
+    }
   }
 
   return hbsString

package/src/automations/logging/index.ts

@@ -0,0 +1,35 @@
+import * as env from "../../environment"
+import { AutomationResults, Automation, App } from "@budibase/types"
+import { automations } from "@budibase/pro"
+import { db as dbUtils } from "@budibase/backend-core"
+
+export async function storeLog(
+  automation: Automation,
+  results: AutomationResults
+) {
+  // can disable this if un-needed in self-host, also only do this for prod apps
+  if (env.DISABLE_AUTOMATION_LOGS) {
+    return
+  }
+  await automations.logs.storeLog(automation, results)
+}
+
+export async function checkAppMetadata(apps: App[]) {
+  const maxStartDate = await automations.logs.oldestLogDate()
+  for (let metadata of apps) {
+    if (!metadata.automationErrors) {
+      continue
+    }
+    for (let [key, errors] of Object.entries(metadata.automationErrors)) {
+      const updated = []
+      for (let error of errors) {
+        const startDate = error.split(dbUtils.SEPARATOR)[2]
+        if (startDate > maxStartDate) {
+          updated.push(error)
+        }
+      }
+      metadata.automationErrors[key] = updated
+    }
+  }
+  return apps
+}

package/src/automations/steps/executeQuery.js

@@ -72,7 +72,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await queryController.executeV2(ctx)
+    await queryController.executeV2(ctx, { isAutomation: true })
     const { data, ...rest } = ctx.body
     return {
       response: data,

package/src/automations/steps/filter.js

@@ -50,43 +50,51 @@ exports.definition = {
     outputs: {
       properties: {
         success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+        result: {
           type: "boolean",
           description: "Whether the logic block passed",
         },
       },
-      required: ["success"],
+      required: ["success", "result"],
     },
   },
 }
 
 exports.run = async function filter({ inputs }) {
-  let { field, condition, value } = inputs
-  // coerce types so that we can use them
-  if (!isNaN(value) && !isNaN(field)) {
-    value = parseFloat(value)
-    field = parseFloat(field)
-  } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
-    value = Date.parse(value)
-    field = Date.parse(field)
-  }
-  let success = false
-  if (typeof field !== "object" && typeof value !== "object") {
-    switch (condition) {
-      case FilterConditions.EQUAL:
-        success = field === value
-        break
-      case FilterConditions.NOT_EQUAL:
-        success = field !== value
-        break
-      case FilterConditions.GREATER_THAN:
-        success = field > value
-        break
-      case FilterConditions.LESS_THAN:
-        success = field < value
-        break
+  try {
+    let { field, condition, value } = inputs
+    // coerce types so that we can use them
+    if (!isNaN(value) && !isNaN(field)) {
+      value = parseFloat(value)
+      field = parseFloat(field)
+    } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
+      value = Date.parse(value)
+      field = Date.parse(field)
+    }
+    let result = false
+    if (typeof field !== "object" && typeof value !== "object") {
+      switch (condition) {
+        case FilterConditions.EQUAL:
+          result = field === value
+          break
+        case FilterConditions.NOT_EQUAL:
+          result = field !== value
+          break
+        case FilterConditions.GREATER_THAN:
+          result = field > value
+          break
+        case FilterConditions.LESS_THAN:
+          result = field < value
+          break
+      }
+    } else {
+      result = false
     }
-  } else {
-    success = false
+    return { success: true, result }
+  } catch (err) {
+    return { success: false, result: false }
   }
-  return { success }
 }

package/src/automations/tests/filter.spec.js

@@ -6,7 +6,8 @@ describe("test the filter logic", () => {
     let res = await setup.runStep(setup.actions.FILTER.stepId,
       { field, condition, value }
     )
-    expect(res.success).toEqual(pass)
+    expect(res.result).toEqual(pass)
+    expect(res.success).toEqual(true)
   }
 
   it("should be able test equality", async () => {

package/src/automations/triggers.js

@@ -97,7 +97,7 @@ exports.externalTrigger = async function (
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
     const fields = automation.definition.trigger.inputs.fields
-    for (let key of Object.keys(fields)) {
+    for (let key of Object.keys(fields || {})) {
       coercedFields[key] = coerce(params.fields[key], fields[key])
     }
     params.fields = coercedFields

package/src/db/linkedRows/index.js

@@ -65,6 +65,7 @@ async function getLinksForRows(rows) {
   // return duplicates, could be querying for both tables in a relation
   return getUniqueByProp(
     responses
+      .filter(el => el != null)
       // create a unique ID which we can use for getting only unique ones
       .map(el => ({ ...el, unique: el.id + el.thisId + el.fieldName })),
     "unique"