@budibase/server 1.0.212-alpha.11 → 1.0.212-alpha.14

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.212-alpha.11",
+  "version": "1.0.212-alpha.14",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.0.212-alpha.11",
-    "@budibase/client": "^1.0.212-alpha.11",
-    "@budibase/pro": "1.0.212-alpha.10",
-    "@budibase/string-templates": "^1.0.212-alpha.11",
-    "@budibase/types": "^1.0.212-alpha.11",
+    "@budibase/backend-core": "^1.0.212-alpha.14",
+    "@budibase/client": "^1.0.212-alpha.14",
+    "@budibase/pro": "1.0.212-alpha.13",
+    "@budibase/string-templates": "^1.0.212-alpha.14",
+    "@budibase/types": "^1.0.212-alpha.14",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -195,5 +195,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "d123d0b4fd3a7b449f759f8841645987c4acc013"
+  "gitHead": "5147929cb55fe0ad5fa21adf64774458b7176c7e"
 }

package/src/api/controllers/application.ts

@@ -45,10 +45,12 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 import { syncGlobalUsers } from "./user"
 const { app: appCache } = require("@budibase/backend-core/cache")
 import { cleanupAutomations } from "../../automations/utils"
+import { checkAppMetadata } from "../../automations/logging"
 const {
   getAppDB,
   getProdAppDB,
   updateAppId,
+  doInAppContext,
 } = require("@budibase/backend-core/context")
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
@@ -192,7 +194,7 @@ export const fetch = async (ctx: any) => {
     }
   }
 
-  ctx.body = apps
+  ctx.body = await checkAppMetadata(apps)
 }
 
 export const fetchAppDefinition = async (ctx: any) => {
@@ -548,22 +550,24 @@ export const sync = async (ctx: any, next: any) => {
 }
 
 const updateAppPackage = async (appPackage: any, appId: any) => {
-  const db = getAppDB()
-  const application = await db.get(DocumentTypes.APP_METADATA)
+  return doInAppContext(appId, async () => {
+    const db = getAppDB()
+    const application = await db.get(DocumentTypes.APP_METADATA)
 
-  const newAppPackage = { ...application, ...appPackage }
-  if (appPackage._rev !== application._rev) {
-    newAppPackage._rev = application._rev
-  }
+    const newAppPackage = { ...application, ...appPackage }
+    if (appPackage._rev !== application._rev) {
+      newAppPackage._rev = application._rev
+    }
 
-  // the locked by property is attached by server but generated from
-  // Redis, shouldn't ever store it
-  delete newAppPackage.lockedBy
+    // the locked by property is attached by server but generated from
+    // Redis, shouldn't ever store it
+    delete newAppPackage.lockedBy
 
-  await db.put(newAppPackage)
-  // remove any cached metadata, so that it will be updated
-  await appCache.invalidateAppMetadata(appId)
-  return newAppPackage
+    await db.put(newAppPackage)
+    // remove any cached metadata, so that it will be updated
+    await appCache.invalidateAppMetadata(appId)
+    return newAppPackage
+  })
 }
 
 const createEmptyAppPackage = async (ctx: any, app: any) => {

package/src/api/controllers/automation.js

@@ -1,6 +1,10 @@
 const actions = require("../../automations/actions")
 const triggers = require("../../automations/triggers")
-const { getAutomationParams, generateAutomationID } = require("../../db/utils")
+const {
+  getAutomationParams,
+  generateAutomationID,
+  DocumentTypes,
+} = require("../../db/utils")
 const {
   checkForWebhooks,
   updateTestHistory,
@@ -9,8 +13,14 @@ const {
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
-const { getAppDB } = require("@budibase/backend-core/context")
+const {
+  getAppDB,
+  getProdAppDB,
+  doInAppContext,
+} = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
+const { app } = require("@budibase/backend-core/cache")
+const { automations } = require("@budibase/pro")
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -183,6 +193,29 @@ exports.destroy = async function (ctx) {
   await events.automation.deleted(oldAutomation)
 }
 
+exports.logSearch = async function (ctx) {
+  ctx.body = await automations.logs.logSearch(ctx.request.body)
+}
+
+exports.clearLogError = async function (ctx) {
+  const { automationId, appId } = ctx.request.body
+  await doInAppContext(appId, async () => {
+    const db = getProdAppDB()
+    const metadata = await db.get(DocumentTypes.APP_METADATA)
+    if (!automationId) {
+      delete metadata.automationErrors
+    } else if (
+      metadata.automationErrors &&
+      metadata.automationErrors[automationId]
+    ) {
+      delete metadata.automationErrors[automationId]
+    }
+    await db.put(metadata)
+    await app.invalidateAppMetadata(metadata.appId, metadata)
+    ctx.body = { message: `Error logs cleared.` }
+  })
+}
+
 exports.getActionList = async function (ctx) {
   ctx.body = ACTION_DEFS
 }

package/src/api/controllers/query/index.ts

@@ -129,6 +129,9 @@ export async function preview(ctx: any) {
         parameters,
         transformer,
         queryId,
+        ctx: {
+          user: ctx.user,
+        },
       })
 
     const { rows, keys, info, extra } = await quotas.addQuery(runFn)
@@ -172,6 +175,9 @@ async function execute(ctx: any, opts = { rowsOnly: false }) {
         parameters: enrichedParameters,
         transformer: query.transformer,
         queryId: ctx.params.queryId,
+        ctx: {
+          user: ctx.user,
+        },
       })
 
     const { rows, pagination, extra } = await quotas.addQuery(runFn)

package/src/api/index.js

@@ -72,7 +72,10 @@ router.use(async (ctx, next) => {
       error,
     }
     ctx.log.error(err)
-    console.trace(err)
+    // unauthorised errors don't provide a useful trace
+    if (!env.isTest()) {
+      console.trace(err)
+    }
   }
 })
 

package/src/api/routes/automation.js

@@ -51,6 +51,16 @@ router
     automationValidator(false),
     controller.create
   )
+  .post(
+    "/api/automations/logs/search",
+    authorized(BUILDER),
+    controller.logSearch
+  )
+  .delete(
+    "/api/automations/logs",
+    authorized(BUILDER),
+    controller.clearLogError
+  )
   .delete(
     "/api/automations/:id/:rev",
     paramResource("id"),

package/src/api/routes/tests/query.spec.js

@@ -346,4 +346,170 @@ describe("/queries", () => {
       expect(contents).toBe(null)
     })
   })
+
+  describe("Current User Request Mapping", () => {
+    
+    async function previewGet(datasource, fields, params) {
+      return config.previewQuery(request, config, datasource, fields, params)
+    }
+
+    async function previewPost(datasource, fields, params) {
+      return config.previewQuery(request, config, datasource, fields, params, "create")
+    }
+
+    it("should parse global and query level header mappings", async () => {
+      const userDetails = config.getUserDetails()
+
+      const datasource = await config.restDatasource({
+        defaultHeaders: {
+          "test": "headerVal",
+          "emailHdr": "{{[user].[email]}}"
+        }
+      })
+      const res = await previewGet(datasource, {
+        path: "www.google.com",
+        queryString: "email={{[user].[email]}}",
+        headers: {
+          queryHdr : "{{[user].[firstName]}}",
+          secondHdr : "1234"
+        }
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      expect(parsedRequest.opts.headers).toEqual({
+        "test": "headerVal",
+        "emailHdr": userDetails.email,
+        "queryHdr": userDetails.firstName,
+        "secondHdr" : "1234"
+      })
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email)
+    })
+
+    it("should bind the current user to query parameters", async () => {
+      const userDetails = config.getUserDetails()
+  
+      const datasource = await config.restDatasource()
+  
+      const res = await previewGet(datasource, {
+        path: "www.google.com",
+        queryString: "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
+      }, {
+        "myEmail" : "{{[user].[email]}}",
+        "myName" : "{{[user].[firstName]}}",
+        "testParam" : "1234"
+      })
+  
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email + 
+        "&testName=" + userDetails.firstName + "&testParam=1234")
+    })
+
+    it("should bind the current user the request body - plain text", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
+        bodyType: "text"
+      }, {
+        "testParam" : "1234"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      expect(parsedRequest.opts.body).toEqual(`This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - json", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "json"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+   
+    it("should bind the current user the request body - xml", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "<note> <email>{{[user].[email]}}</email> <code>{{testParam}}</code> " + 
+          "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
+        bodyType: "xml"
+      }, {
+        "testParam" : "1234",
+        "userId" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
+        
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - form-data", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "form"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+
+      const emailData = parsedRequest.opts.body._streams[1]
+      expect(emailData).toEqual(userDetails.email)
+
+      const queryCodeData = parsedRequest.opts.body._streams[4]
+      expect(queryCodeData).toEqual("1234")
+
+      const userRef = parsedRequest.opts.body._streams[7]
+      expect(userRef).toEqual(userDetails.firstName)
+
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234")
+    })
+
+    it("should bind the current user the request body - encoded", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+  
+      const res = await previewPost(datasource, {
+        path: "www.google.com",
+        queryString: "testParam={{testParam}}",
+        requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}",
+        bodyType: "encoded"
+      }, {
+        "testParam" : "1234",
+        "userRef" : "{{[user].[firstName]}}"
+      })
+      const parsedRequest = JSON.parse(res.body.extra.raw)
+      
+      expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
+      expect(parsedRequest.opts.body.queryCode).toEqual("1234")
+      expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName)
+    })
+
+  });
 })

package/src/automations/logging/index.ts

@@ -0,0 +1,35 @@
+import * as env from "../../environment"
+import { AutomationResults, Automation, App } from "@budibase/types"
+import { automations } from "@budibase/pro"
+import { db as dbUtils } from "@budibase/backend-core"
+
+export async function storeLog(
+  automation: Automation,
+  results: AutomationResults
+) {
+  // can disable this if un-needed in self-host, also only do this for prod apps
+  if (env.DISABLE_AUTOMATION_LOGS) {
+    return
+  }
+  await automations.logs.storeLog(automation, results)
+}
+
+export async function checkAppMetadata(apps: App[]) {
+  const maxStartDate = await automations.logs.oldestLogDate()
+  for (let metadata of apps) {
+    if (!metadata.automationErrors) {
+      continue
+    }
+    for (let [key, errors] of Object.entries(metadata.automationErrors)) {
+      const updated = []
+      for (let error of errors) {
+        const startDate = error.split(dbUtils.SEPARATOR)[2]
+        if (startDate > maxStartDate) {
+          updated.push(error)
+        }
+      }
+      metadata.automationErrors[key] = updated
+    }
+  }
+  return apps
+}

package/src/automations/steps/filter.js

@@ -50,43 +50,51 @@ exports.definition = {
     outputs: {
       properties: {
         success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+        result: {
           type: "boolean",
           description: "Whether the logic block passed",
         },
       },
-      required: ["success"],
+      required: ["success", "result"],
     },
   },
 }
 
 exports.run = async function filter({ inputs }) {
-  let { field, condition, value } = inputs
-  // coerce types so that we can use them
-  if (!isNaN(value) && !isNaN(field)) {
-    value = parseFloat(value)
-    field = parseFloat(field)
-  } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
-    value = Date.parse(value)
-    field = Date.parse(field)
-  }
-  let success = false
-  if (typeof field !== "object" && typeof value !== "object") {
-    switch (condition) {
-      case FilterConditions.EQUAL:
-        success = field === value
-        break
-      case FilterConditions.NOT_EQUAL:
-        success = field !== value
-        break
-      case FilterConditions.GREATER_THAN:
-        success = field > value
-        break
-      case FilterConditions.LESS_THAN:
-        success = field < value
-        break
+  try {
+    let { field, condition, value } = inputs
+    // coerce types so that we can use them
+    if (!isNaN(value) && !isNaN(field)) {
+      value = parseFloat(value)
+      field = parseFloat(field)
+    } else if (!isNaN(Date.parse(value)) && !isNaN(Date.parse(field))) {
+      value = Date.parse(value)
+      field = Date.parse(field)
+    }
+    let result = false
+    if (typeof field !== "object" && typeof value !== "object") {
+      switch (condition) {
+        case FilterConditions.EQUAL:
+          result = field === value
+          break
+        case FilterConditions.NOT_EQUAL:
+          result = field !== value
+          break
+        case FilterConditions.GREATER_THAN:
+          result = field > value
+          break
+        case FilterConditions.LESS_THAN:
+          result = field < value
+          break
+      }
+    } else {
+      result = false
     }
-  } else {
-    success = false
+    return { success: true, result }
+  } catch (err) {
+    return { success: false, result: false }
   }
-  return { success }
 }

package/src/automations/tests/filter.spec.js

@@ -6,7 +6,8 @@ describe("test the filter logic", () => {
     let res = await setup.runStep(setup.actions.FILTER.stepId,
       { field, condition, value }
     )
-    expect(res.success).toEqual(pass)
+    expect(res.result).toEqual(pass)
+    expect(res.success).toEqual(true)
   }
 
   it("should be able test equality", async () => {

package/src/db/linkedRows/index.js

@@ -65,6 +65,7 @@ async function getLinksForRows(rows) {
   // return duplicates, could be querying for both tables in a relation
   return getUniqueByProp(
     responses
+      .filter(el => el != null)
       // create a unique ID which we can use for getting only unique ones
       .map(el => ({ ...el, unique: el.id + el.thisId + el.fieldName })),
     "unique"

package/src/db/utils.js

@@ -11,6 +11,8 @@ const {
   isProdAppID,
   getDevelopmentAppID,
   generateAppID,
+  getQueryIndex,
+  ViewNames,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -22,11 +24,7 @@ const AppStatus = {
 }
 
 const DocumentTypes = {
-  APP: CoreDocTypes.APP,
-  DEV: CoreDocTypes.DEV,
-  APP_DEV: CoreDocTypes.APP_DEV,
-  APP_METADATA: CoreDocTypes.APP_METADATA,
-  ROLE: CoreDocTypes.ROLE,
+  ...CoreDocTypes,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
@@ -45,11 +43,6 @@ const DocumentTypes = {
   USER_FLAG: "flag",
 }
 
-const ViewNames = {
-  LINK: "by_link",
-  ROUTING: "screen_routes",
-}
-
 const InternalTables = {
   USER_METADATA: "ta_users",
 }
@@ -89,9 +82,7 @@ exports.generateDevAppID = getDevelopmentAppID
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
 
-exports.getQueryIndex = viewName => {
-  return `database/${viewName}`
-}
+exports.getQueryIndex = getQueryIndex
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this

package/src/environment.js

@@ -78,6 +78,7 @@ module.exports = {
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,

package/src/integrations/queries/sql.ts

@@ -9,7 +9,9 @@ export function enrichQueryFields(
   parameters = {}
 ) {
   const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
-
+  if (!fields || !parameters) {
+    return enrichedQuery
+  }
   // enrich the fields with dynamic parameters
   for (let key of Object.keys(fields)) {
     if (fields[key] == null) {

package/src/integrations/rest.ts

@@ -287,7 +287,7 @@ module RestModule {
           input.body = form
           break
         case BodyTypes.XML:
-          if (object != null) {
+          if (object != null && Object.keys(object).length) {
             string = new XmlBuilder().buildObject(object)
           }
           input.body = string

package/src/integrations/tests/rest.spec.js

@@ -155,12 +155,27 @@ describe("REST Integration", () => {
       expect(output.headers["Content-Type"]).toEqual("application/json")
     })
 
-    it("should allow XML", () => {
+    it("should allow raw XML", () => {
+      const output = config.integration.addBody("xml", "<a>1</a><b>2</b>", {})
+      expect(output.body.includes("<a>1</a>")).toEqual(true)
+      expect(output.body.includes("<b>2</b>")).toEqual(true)
+      expect(output.headers["Content-Type"]).toEqual("application/xml")
+    })
+
+    it("should allow a valid js object and parse the contents to xml", () => {
       const output = config.integration.addBody("xml", input, {})
       expect(output.body.includes("<a>1</a>")).toEqual(true)
       expect(output.body.includes("<b>2</b>")).toEqual(true)
       expect(output.headers["Content-Type"]).toEqual("application/xml")
     })
+
+    it("should allow a valid json string and parse the contents to xml", () => {
+      const output = config.integration.addBody("xml", JSON.stringify(input), {})
+      expect(output.body.includes("<a>1</a>")).toEqual(true)
+      expect(output.body.includes("<b>2</b>")).toEqual(true)
+      expect(output.headers["Content-Type"]).toEqual("application/xml")
+    })
+
   })
 
   describe("response", () => {

package/src/migrations/tests/index.spec.ts

@@ -93,8 +93,24 @@ describe("migrations", () => {
       await clearMigrations()
       const appId = config.prodAppId
       const roles = { [appId]: "role_12345" }
-      await config.createUser(undefined, undefined, false, true, roles) // admin only
-      await config.createUser(undefined, undefined, false, false, roles) // non admin non builder
+      await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        true,
+        roles
+      ) // admin only
+      await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        false,
+        roles
+      ) // non admin non builder
       await config.createTable()
       await config.createRow()
       await config.createRow()

package/src/module.d.ts

@@ -8,3 +8,4 @@ declare module "@budibase/backend-core/constants"
 declare module "@budibase/backend-core/auth"
 declare module "@budibase/backend-core/sessions"
 declare module "@budibase/backend-core/encryption"
+declare module "@budibase/backend-core/redis"