@budibase/server 1.0.217 → 1.0.219-alpha.1

package/src/integrations/tests/rest.spec.js

@@ -155,12 +155,27 @@ describe("REST Integration", () => {
       expect(output.headers["Content-Type"]).toEqual("application/json")
     })
 
-    it("should allow XML", () => {
+    it("should allow raw XML", () => {
+      const output = config.integration.addBody("xml", "<a>1</a><b>2</b>", {})
+      expect(output.body.includes("<a>1</a>")).toEqual(true)
+      expect(output.body.includes("<b>2</b>")).toEqual(true)
+      expect(output.headers["Content-Type"]).toEqual("application/xml")
+    })
+
+    it("should allow a valid js object and parse the contents to xml", () => {
       const output = config.integration.addBody("xml", input, {})
       expect(output.body.includes("<a>1</a>")).toEqual(true)
       expect(output.body.includes("<b>2</b>")).toEqual(true)
       expect(output.headers["Content-Type"]).toEqual("application/xml")
     })
+
+    it("should allow a valid json string and parse the contents to xml", () => {
+      const output = config.integration.addBody("xml", JSON.stringify(input), {})
+      expect(output.body.includes("<a>1</a>")).toEqual(true)
+      expect(output.body.includes("<b>2</b>")).toEqual(true)
+      expect(output.headers["Content-Type"]).toEqual("application/xml")
+    })
+
   })
 
   describe("response", () => {

package/src/migrations/tests/index.spec.ts

@@ -93,8 +93,24 @@ describe("migrations", () => {
       await clearMigrations()
       const appId = config.prodAppId
       const roles = { [appId]: "role_12345" }
-      await config.createUser(undefined, undefined, false, true, roles) // admin only
-      await config.createUser(undefined, undefined, false, false, roles) // non admin non builder
+      await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        true,
+        roles
+      ) // admin only
+      await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        false,
+        roles
+      ) // non admin non builder
       await config.createTable()
       await config.createRow()
       await config.createRow()

package/src/module.d.ts

@@ -8,3 +8,5 @@ declare module "@budibase/backend-core/constants"
 declare module "@budibase/backend-core/auth"
 declare module "@budibase/backend-core/sessions"
 declare module "@budibase/backend-core/encryption"
+declare module "@budibase/backend-core/utils"
+declare module "@budibase/backend-core/redis"

package/src/tests/utilities/TestConfiguration.js

@@ -28,6 +28,8 @@ const { encrypt } = require("@budibase/backend-core/encryption")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
+const FIRSTNAME = "Barbara"
+const LASTNAME = "Barbington"
 const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306"
 
 class TestConfiguration {
@@ -59,6 +61,15 @@ class TestConfiguration {
     return this.prodAppId
   }
 
+  getUserDetails() {
+    return {
+      globalId: GLOBAL_USER_ID,
+      email: EMAIL,
+      firstName: FIRSTNAME,
+      lastName: LASTNAME,
+    }
+  }
+
   async doInContext(appId, task) {
     if (!appId) {
       appId = this.appId
@@ -118,6 +129,8 @@ class TestConfiguration {
   // USER / AUTH
   async globalUser({
     id = GLOBAL_USER_ID,
+    firstName = FIRSTNAME,
+    lastName = LASTNAME,
     builder = true,
     admin = false,
     email = EMAIL,
@@ -135,6 +148,8 @@ class TestConfiguration {
         ...existing,
         roles: roles || {},
         tenantId: TENANT_ID,
+        firstName,
+        lastName,
       }
       await createASession(id, {
         sessionId: "sessionid",
@@ -161,6 +176,8 @@ class TestConfiguration {
 
   async createUser(
     id = null,
+    firstName = FIRSTNAME,
+    lastName = LASTNAME,
     email = EMAIL,
     builder = true,
     admin = false,
@@ -169,6 +186,8 @@ class TestConfiguration {
     const globalId = !id ? `us_${Math.random()}` : `us_${id}`
     const resp = await this.globalUser({
       id: globalId,
+      firstName,
+      lastName,
       email,
       builder,
       admin,
@@ -520,14 +539,14 @@ class TestConfiguration {
 
   // QUERY
 
-  async previewQuery(request, config, datasource, fields) {
+  async previewQuery(request, config, datasource, fields, params, verb) {
     return request
       .post(`/api/queries/preview`)
       .send({
         datasourceId: datasource._id,
-        parameters: {},
+        parameters: params || {},
         fields,
-        queryVerb: "read",
+        queryVerb: verb || "read",
         name: datasource.name,
       })
       .set(config.defaultHeaders())

package/src/threads/automation.js

@@ -9,11 +9,12 @@ const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { AutomationErrors, LoopStepTypes } = require("../constants")
+const { storeLog } = require("../automations/logging")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 
 const CRON_STEP_ID = triggerDefs.CRON.stepId
-const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const STOPPED_STATUS = { success: true, status: "STOPPED" }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
@@ -218,7 +219,7 @@ class Orchestrator {
             }
           }
           if (
-            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === env.AUTOMATION_MAX_ITERATION ||
             index === parseInt(loopStep.inputs.iterations)
           ) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
@@ -275,7 +276,7 @@ class Orchestrator {
           this._context.steps[stepCount] = outputs
           // if filter causes us to stop execution don't break the loop, set a var
           // so that we can finish iterating through the steps and record that it stopped
-          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+          if (step.stepId === FILTER_STEP_ID && !outputs.result) {
             stopped = true
             this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
               ...outputs,
@@ -325,6 +326,8 @@ class Orchestrator {
       }
     }
 
+    // store the logs for the automation run
+    await storeLog(this._automation, this.executionOutput)
     return this.executionOutput
   }
 }

package/src/threads/index.ts

@@ -77,7 +77,7 @@ export class Thread {
     })
   }
 
-  static shutdown() {
+  static stopThreads() {
     return new Promise<void>(resolve => {
       if (Thread.workerRefs.length === 0) {
         resolve()
@@ -95,4 +95,8 @@ export class Thread {
       Thread.workerRefs = []
     })
   }
+
+  static async shutdown() {
+    await Thread.stopThreads()
+  }
 }

package/src/threads/query.js

@@ -4,6 +4,12 @@ const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const {
+  refreshOAuthToken,
+  updateUserOAuth,
+} = require("@budibase/backend-core/auth")
+const { getGlobalIDFromUserMetadataID } = require("../db/utils")
+
 const { isSQL } = require("../integrations/utils")
 const {
   enrichQueryFields,
@@ -21,29 +27,56 @@ class QueryRunner {
     this.queryId = input.queryId
     this.noRecursiveQuery = flags.noRecursiveQuery
     this.cachedVariables = []
+    // Additional context items for enrichment
+    this.ctx = input.ctx
     // allows the response from a query to be stored throughout this
     // execution so that if it needs to be re-used for another variable
     // it can be
     this.queryResponse = {}
     this.hasRerun = false
+    this.hasRefreshedOAuth = false
   }
 
   async execute() {
     let { datasource, fields, queryVerb, transformer } = this
+
     const Integration = integrations[datasource.source]
     if (!Integration) {
       throw "Integration type does not exist."
     }
+
+    if (datasource.config.authConfigs) {
+      datasource.config.authConfigs = datasource.config.authConfigs.map(
+        config => {
+          return enrichQueryFields(config, this.ctx)
+        }
+      )
+    }
+
     const integration = new Integration(datasource.config)
 
     // pre-query, make sure datasource variables are added to parameters
     const parameters = await this.addDatasourceVariables()
+
+    // Enrich the parameters with the addition context items.
+    // 'user' is now a reserved variable key in mapping parameters
+    const enrichedParameters = enrichQueryFields(parameters, this.ctx)
+    const enrichedContext = { ...enrichedParameters, ...this.ctx }
+
+    // Parse global headers
+    if (datasource.config.defaultHeaders) {
+      datasource.config.defaultHeaders = enrichQueryFields(
+        datasource.config.defaultHeaders,
+        enrichedContext
+      )
+    }
+
     let query
     // handle SQL injections by interpolating the variables
     if (isSQL(datasource)) {
-      query = interpolateSQL(fields, parameters, integration)
+      query = interpolateSQL(fields, enrichedParameters, integration)
     } else {
-      query = enrichQueryFields(fields, parameters)
+      query = enrichQueryFields(fields, enrichedContext)
     }
 
     // Add pagination values for REST queries
@@ -67,20 +100,30 @@ class QueryRunner {
     if (transformer) {
       const runner = new ScriptRunner(transformer, {
         data: rows,
-        params: parameters,
+        params: enrichedParameters,
       })
       rows = runner.execute()
     }
 
     // if the request fails we retry once, invalidating the cached value
-    if (
-      info &&
-      info.code >= 400 &&
-      this.cachedVariables.length > 0 &&
-      !this.hasRerun
-    ) {
+    if (info && info.code >= 400 && !this.hasRerun) {
+      if (
+        this.ctx.user?.provider &&
+        info.code === 401 &&
+        !this.hasRefreshedOAuth
+      ) {
+        // Attempt to refresh the access token from the provider
+        this.hasRefreshedOAuth = true
+        const authResponse = await this.refreshOAuth2(this.ctx)
+
+        if (!authResponse || authResponse.err) {
+          // In this event the user may have oAuth issues that
+          // could require re-authenticating with their provider.
+          throw new Error("OAuth2 access token could not be refreshed")
+        }
+      }
+
       this.hasRerun = true
-      // invalidate the cache value
       await threadUtils.invalidateDynamicVariables(this.cachedVariables)
       return this.execute()
     }
@@ -126,6 +169,31 @@ class QueryRunner {
     ).execute()
   }
 
+  async refreshOAuth2(ctx) {
+    const { oauth2, providerType, _id } = ctx.user
+    const { configId } = ctx.auth
+
+    if (!providerType || !oauth2?.refreshToken) {
+      console.error("No refresh token found for authenticated user")
+      return
+    }
+
+    const resp = await refreshOAuthToken(
+      oauth2.refreshToken,
+      providerType,
+      configId
+    )
+
+    // Refresh session flow. Should be in same location as refreshOAuthToken
+    // There are several other properties available in 'resp'
+    if (!resp.error) {
+      const globalUserId = getGlobalIDFromUserMetadataID(_id)
+      await updateUserOAuth(globalUserId, resp)
+    }
+
+    return resp
+  }
+
   async getDynamicVariable(variable) {
     let { parameters } = this
     const queryId = variable.queryId,

package/src/utilities/fileSystem/index.js

@@ -21,6 +21,7 @@ const env = require("../../environment")
 const {
   USER_METDATA_PREFIX,
   LINK_USER_METADATA_PREFIX,
+  TABLE_ROW_PREFIX,
 } = require("../../db/utils")
 const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
@@ -109,6 +110,23 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
+exports.defineFilter = excludeRows => {
+  if (excludeRows) {
+    return doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc._id.includes(LINK_USER_METADATA_PREFIX) ||
+        doc._id.includes(TABLE_ROW_PREFIX)
+      )
+  } else if (!excludeRows) {
+    return doc =>
+      !(
+        doc._id.includes(USER_METDATA_PREFIX) ||
+        doc._id.includes(LINK_USER_METADATA_PREFIX)
+      )
+  }
+}
+
 /**
  * Local utility to back up the database state for an app, excluding global user
  * data or user relationships.
@@ -116,14 +134,10 @@ exports.apiFileReturn = contents => {
  * @param {object} config Config to send to export DB
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = async (appId, config) => {
+const backupAppData = async (appId, config, includeRows) => {
   return await exports.exportDB(appId, {
     ...config,
-    filter: doc =>
-      !(
-        doc._id.includes(USER_METDATA_PREFIX) ||
-        doc._id.includes(LINK_USER_METADATA_PREFIX)
-      ),
+    filter: exports.defineFilter(includeRows),
   })
 }
 
@@ -142,8 +156,8 @@ exports.performBackup = async (appId, backupName) => {
  * @param {string} appId The ID of the app which is to be backed up.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = async appId => {
-  return await backupAppData(appId, { stream: true })
+exports.streamBackup = async (appId, includeRows) => {
+  return await backupAppData(appId, { stream: true }, includeRows)
 }
 
 /**