@budibase/server 1.0.219 → 1.0.220-alpha.0

package/src/api/index.js

@@ -72,7 +72,10 @@ router.use(async (ctx, next) => {
       error,
     }
     ctx.log.error(err)
-    console.trace(err)
+    // unauthorised errors don't provide a useful trace
+    if (!env.isTest()) {
+      console.trace(err)
+    }
   }
 })
 

package/src/api/routes/automation.js

@@ -51,6 +51,16 @@ router
     automationValidator(false),
     controller.create
   )
+  .post(
+    "/api/automations/logs/search",
+    authorized(BUILDER),
+    controller.logSearch
+  )
+  .delete(
+    "/api/automations/logs",
+    authorized(BUILDER),
+    controller.clearLogError
+  )
   .delete(
     "/api/automations/:id/:rev",
     paramResource("id"),

package/src/api/routes/public/rows.ts

@@ -150,6 +150,11 @@ read.push(new Endpoint("get", "/tables/:tableId/rows/:rowId", controller.read))
  *               query:
  *                 type: object
  *                 properties:
+ *                   allOr:
+ *                     type: boolean
+ *                     description: Specifies that a row should be returned if it satisfies
+ *                       any of the specified options, rather than requiring it to fulfill all
+ *                       the search parameters. This defaults to false, meaning AND logic will be used.
  *                   string:
  *                     type: object
  *                     example:
@@ -167,9 +172,9 @@ read.push(new Endpoint("get", "/tables/:tableId/rows/:rowId", controller.read))
  *                   range:
  *                     type: object
  *                     description: Searches within a range, the format of this must be
- *                       columnName -> [low, high].
+ *                       in the format of an object with a "low" and "high" property.
  *                     example:
- *                       columnName1: [10, 20]
+ *                       columnName1: { low: 10, high: 20 }
  *                   equal:
  *                     type: object
  *                     description: Searches for rows that have a column value that is

package/src/api/routes/tests/query.spec.js

@@ -218,7 +218,6 @@ describe("/queries", () => {
       expect(res.body.schemaFields).toEqual(["a", "b"])
       expect(res.body.rows.length).toEqual(1)
       expect(events.query.previewed).toBeCalledTimes(1)
-      datasource.config = { schema: "public" }
       expect(events.query.previewed).toBeCalledWith(datasource, query)
     })
 
@@ -346,4 +345,170 @@ describe("/queries", () => {
       expect(contents).toBe(null)
     })
   })
+
+  describe("Current User Request Mapping", () => {
+    
+    async function previewGet(datasource, fields, params) {
+      return config.previewQuery(request, config, datasource, fields, params)
+    }
+
+    async function previewPost(datasource, fields, params) {
+      return config.previewQuery(request, config, datasource, fields, params, "create")
+    }
+
+    it("should parse global and query level header mappings", async () => {
+      const userDetails = config.getUserDetails()
+
+      const datasource = await config.restDatasource({
+        defaultHeaders: {
+          "test": "headerVal",
+          "emailHdr": "{{[user].[email]}}"
+        }
+      })
+      const res = await previewGet(datasource, {
+        path: "www.google.com",
+        queryString: "email={{[user].[email]}}",
+        headers: {
+          queryHdr : "{{[user].[firstName]}}",
+          secondHdr : "1234"
+        }
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      expect(parsedRequest.opts.headers).toEqual({
+        "test": "headerVal",
+        "emailHdr": userDetails.email,
+        "queryHdr": userDetails.firstName,
+        "secondHdr" : "1234"
+      })
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email)
+    })
+
+    it("should bind the current user to query parameters", async () => {
+      const userDetails = config.getUserDetails()
+  
+      const datasource = await config.restDatasource()
+  
+      const res = await previewGet(datasource, {
+        path: "www.google.com",
+        queryString: "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
+      }, {
+        "myEmail" : "{{[user].[email]}}",
+        "myName" : "{{[user].[firstName]}}",
+        "testParam" : "1234"
+      })
+  
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email + 
+        "&testName=" + userDetails.firstName + "&testParam=1234")
+    })
+
+    it("should bind the current user the request body - plain text", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
+        bodyType: "text"
+      }, {
+        "testParam" : "1234"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      expect(parsedRequest.opts.body).toEqual(`This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - json", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "json"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+   
+    it("should bind the current user the request body - xml", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "<note> <email>{{[user].[email]}}</email> <code>{{testParam}}</code> " + 
+          "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
+        bodyType: "xml"
+      }, {
+        "testParam" : "1234",
+        "userId" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
+        
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - form-data", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "form"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+
+      const emailData = parsedRequest.opts.body._streams[1]
+      expect(emailData).toEqual(userDetails.email)
+
+      const queryCodeData = parsedRequest.opts.body._streams[4]
+      expect(queryCodeData).toEqual("1234")
+
+      const userRef = parsedRequest.opts.body._streams[7]
+      expect(userRef).toEqual(userDetails.firstName)
+
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - encoded", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "encoded"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      
+      expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
+      expect(parsedRequest.opts.body.queryCode).toEqual("1234")
+      expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName)
+    })
+
+  });
 })

package/src/api/routes/utils/validators.js

@@ -58,6 +58,7 @@ function filterObject() {
     oneOf: Joi.object().optional(),
     contains: Joi.object().optional(),
     notContains: Joi.object().optional(),
+    allOr: Joi.boolean().optional(),
   }).unknown(true)
 }
 

package/src/app.ts

@@ -18,7 +18,9 @@ const { logAlert } = require("@budibase/backend-core/logging")
 const { Thread } = require("./threads")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
-import { events, installation } from "@budibase/backend-core"
+import { events, installation, tenancy } from "@budibase/backend-core"
+import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { tenantSucceeded } from "@budibase/backend-core/dist/src/events/publishers/backfill"
 
 const app = new Koa()
 
@@ -110,6 +112,33 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
+  // check and create admin user if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.BB_ADMIN_USER_EMAIL &&
+    env.BB_ADMIN_USER_PASSWORD
+  ) {
+    const checklist = await getChecklist()
+    if (!checklist?.adminUser?.checked) {
+      try {
+        const tenantId = tenancy.getTenantId()
+        await createAdminUser(
+          env.BB_ADMIN_USER_EMAIL,
+          env.BB_ADMIN_USER_PASSWORD,
+          tenantId
+        )
+        console.log(
+          "Admin account automatically created for",
+          env.BB_ADMIN_USER_EMAIL
+        )
+      } catch (e) {
+        logAlert("Error creating initial admin user. Exiting.", e)
+        shutdown()
+      }
+    }
+  }
+
   // check for version updates
   await installation.checkInstallVersion()
 

package/src/automations/automationUtils.js

@@ -1,5 +1,10 @@
 const { getTable } = require("../api/controllers/table/utils")
-const { findHBSBlocks } = require("@budibase/string-templates")
+const {
+  findHBSBlocks,
+  decodeJSBinding,
+  isJSBinding,
+  encodeJSBinding,
+} = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -77,11 +82,21 @@ exports.getError = err => {
 }
 
 exports.substituteLoopStep = (hbsString, substitute) => {
-  let blocks = findHBSBlocks(hbsString)
+  let blocks = []
+  let checkForJS = isJSBinding(hbsString)
+  if (checkForJS) {
+    hbsString = decodeJSBinding(hbsString)
+    blocks.push(hbsString)
+  } else {
+    blocks = findHBSBlocks(hbsString)
+  }
   for (let block of blocks) {
-    let oldBlock = block
     block = block.replace(/loop/, substitute)
-    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
+    if (checkForJS) {
+      hbsString = encodeJSBinding(block)
+    } else {
+      hbsString = block
+    }
   }
 
   return hbsString

package/src/automations/logging/index.ts

@@ -0,0 +1,35 @@
+import * as env from "../../environment"
+import { AutomationResults, Automation, App } from "@budibase/types"
+import { automations } from "@budibase/pro"
+import { db as dbUtils } from "@budibase/backend-core"
+
+export async function storeLog(
+  automation: Automation,
+  results: AutomationResults
+) {
+  // can disable this if un-needed in self-host, also only do this for prod apps
+  if (env.DISABLE_AUTOMATION_LOGS) {
+    return
+  }
+  await automations.logs.storeLog(automation, results)
+}
+
+export async function checkAppMetadata(apps: App[]) {
+  const maxStartDate = await automations.logs.oldestLogDate()
+  for (let metadata of apps) {
+    if (!metadata.automationErrors) {
+      continue
+    }
+    for (let [key, errors] of Object.entries(metadata.automationErrors)) {
+      const updated = []
+      for (let error of errors) {
+        const startDate = error.split(dbUtils.SEPARATOR)[2]
+        if (startDate > maxStartDate) {
+          updated.push(error)
+        }
+      }
+      metadata.automationErrors[key] = updated
+    }
+  }
+  return apps
+}

package/src/automations/steps/executeQuery.js

@@ -72,7 +72,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await queryController.executeV2(ctx)
+    await queryController.executeV2(ctx, { isAutomation: true })
     const { data, ...rest } = ctx.body
     return {
       response: data,

package/src/automations/steps/filter.js

@@ -50,43 +50,51 @@ exports.definition = {
     outputs: {
       properties: {
         success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+        result: {
           type: "boolean",
           description: "Whether the logic block passed",
         },
       },
-      required: ["success"],
+      required: ["success", "result"],
     },
   },
 }
 
 exports.run = async function filter({ inputs }) {
-  let { field, condition, value } = inputs
-  // coerce types so that we can use them
-  if (!isNaN(value) && !isNaN(field)) {
-    value = parseFloat(value)
-    field = parseFloat(field)
-  } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
-    value = Date.parse(value)
-    field = Date.parse(field)
-  }
-  let success = false
-  if (typeof field !== "object" && typeof value !== "object") {
-    switch (condition) {
-      case FilterConditions.EQUAL:
-        success = field === value
-        break
-      case FilterConditions.NOT_EQUAL:
-        success = field !== value
-        break
-      case FilterConditions.GREATER_THAN:
-        success = field > value
-        break
-      case FilterConditions.LESS_THAN:
-        success = field < value
-        break
+  try {
+    let { field, condition, value } = inputs
+    // coerce types so that we can use them
+    if (!isNaN(value) && !isNaN(field)) {
+      value = parseFloat(value)
+      field = parseFloat(field)
+    } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
+      value = Date.parse(value)
+      field = Date.parse(field)
+    }
+    let result = false
+    if (typeof field !== "object" && typeof value !== "object") {
+      switch (condition) {
+        case FilterConditions.EQUAL:
+          result = field === value
+          break
+        case FilterConditions.NOT_EQUAL:
+          result = field !== value
+          break
+        case FilterConditions.GREATER_THAN:
+          result = field > value
+          break
+        case FilterConditions.LESS_THAN:
+          result = field < value
+          break
+      }
+    } else {
+      result = false
     }
-  } else {
-    success = false
+    return { success: true, result }
+  } catch (err) {
+    return { success: false, result: false }
   }
-  return { success }
 }

package/src/automations/tests/filter.spec.js

@@ -6,7 +6,8 @@ describe("test the filter logic", () => {
     let res = await setup.runStep(setup.actions.FILTER.stepId,
       { field, condition, value }
     )
-    expect(res.success).toEqual(pass)
+    expect(res.result).toEqual(pass)
+    expect(res.success).toEqual(true)
   }
 
   it("should be able test equality", async () => {

package/src/automations/triggers.js

@@ -97,7 +97,7 @@ exports.externalTrigger = async function (
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
     const fields = automation.definition.trigger.inputs.fields
-    for (let key of Object.keys(fields)) {
+    for (let key of Object.keys(fields || {})) {
       coercedFields[key] = coerce(params.fields[key], fields[key])
     }
     params.fields = coercedFields

package/src/db/linkedRows/index.js

@@ -65,6 +65,7 @@ async function getLinksForRows(rows) {
   // return duplicates, could be querying for both tables in a relation
   return getUniqueByProp(
     responses
+      .filter(el => el != null)
       // create a unique ID which we can use for getting only unique ones
       .map(el => ({ ...el, unique: el.id + el.thisId + el.fieldName })),
     "unique"

package/src/db/utils.js

@@ -11,6 +11,8 @@ const {
   isProdAppID,
   getDevelopmentAppID,
   generateAppID,
+  getQueryIndex,
+  ViewNames,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -22,11 +24,7 @@ const AppStatus = {
 }
 
 const DocumentTypes = {
-  APP: CoreDocTypes.APP,
-  DEV: CoreDocTypes.DEV,
-  APP_DEV: CoreDocTypes.APP_DEV,
-  APP_METADATA: CoreDocTypes.APP_METADATA,
-  ROLE: CoreDocTypes.ROLE,
+  ...CoreDocTypes,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
@@ -45,11 +43,6 @@ const DocumentTypes = {
   USER_FLAG: "flag",
 }
 
-const ViewNames = {
-  LINK: "by_link",
-  ROUTING: "screen_routes",
-}
-
 const InternalTables = {
   USER_METADATA: "ta_users",
 }
@@ -74,6 +67,7 @@ exports.isDevAppID = isDevAppID
 exports.isProdAppID = isProdAppID
 exports.USER_METDATA_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
 exports.LINK_USER_METADATA_PREFIX = `${DocumentTypes.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.TABLE_ROW_PREFIX = `${DocumentTypes.ROW}${SEPARATOR}${DocumentTypes.TABLE}`
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
 exports.DocumentTypes = DocumentTypes
@@ -88,9 +82,7 @@ exports.generateDevAppID = getDevelopmentAppID
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
 
-exports.getQueryIndex = viewName => {
-  return `database/${viewName}`
-}
+exports.getQueryIndex = getQueryIndex
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this

package/src/definitions/common.ts

@@ -12,6 +12,7 @@ export interface Application extends Base {
 export interface FieldSchema {
   // TODO: replace with field types enum when done
   type: string
+  externalType?: string
   fieldName?: string
   name: string
   tableId?: string

package/src/definitions/datasource.ts

@@ -50,6 +50,7 @@ export enum SourceNames {
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   FIRESTORE = "FIRESTORE",
   REDIS = "REDIS",
+  SNOWFLAKE = "SNOWFLAKE",
 }
 
 export enum IncludeRelationships {
@@ -93,6 +94,7 @@ export interface Integration {
   relationships?: boolean
   description: string
   friendlyName: string
+  type?: string
   datasource: {}
   query: {
     [key: string]: QueryDefinition