@budibase/server 0.9.169-alpha.9 → 0.9.172

package/src/api/routes/dev.js

@@ -6,16 +6,11 @@ const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
-function redirectPath(path) {
-  router
-    .get(`/api/${path}/:devPath(.*)`, controller.buildRedirectGet(path))
-    .post(`/api/${path}/:devPath(.*)`, controller.buildRedirectPost(path))
-    .delete(`/api/${path}/:devPath(.*)`, controller.buildRedirectDelete(path))
-}
-
 if (env.isDev() || env.isTest()) {
-  redirectPath("global")
-  redirectPath("system")
+  router
+    .get("/api/global/:devPath(.*)", controller.redirectGet)
+    .post("/api/global/:devPath(.*)", controller.redirectPost)
+    .delete("/api/global/:devPath(.*)", controller.redirectDelete)
 }
 
 router

package/src/api/routes/query.js

@@ -31,8 +31,7 @@ function generateQueryValidation() {
     })),
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true),
-    transformer: Joi.string().optional(),
+    schema: Joi.object({}).required().unknown(true)
   }))
 }
 
@@ -43,7 +42,6 @@ function generateQueryPreviewValidation() {
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
-    transformer: Joi.string().optional(),
     parameters: Joi.object({}).required().unknown(true)
   }))
 }

package/src/automations/actions.js

@@ -3,6 +3,7 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const executeScript = require("./steps/executeScript")
+const bash = require("./steps/bash")
 const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const serverLog = require("./steps/serverLog")
@@ -13,7 +14,6 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
-const env = require("../environment")
 
 const ACTION_IMPLS = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
@@ -22,6 +22,7 @@ const ACTION_IMPLS = {
   DELETE_ROW: deleteRow.run,
   OUTGOING_WEBHOOK: outgoingWebhook.run,
   EXECUTE_SCRIPT: executeScript.run,
+  EXECUTE_BASH: bash.run,
   EXECUTE_QUERY: executeQuery.run,
   SERVER_LOG: serverLog.run,
   DELAY: delay.run,
@@ -41,6 +42,7 @@ const ACTION_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
   EXECUTE_SCRIPT: executeScript.definition,
   EXECUTE_QUERY: executeQuery.definition,
+  EXECUTE_BASH: bash.definition,
   SERVER_LOG: serverLog.definition,
   DELAY: delay.definition,
   FILTER: filter.definition,
@@ -52,15 +54,6 @@ const ACTION_DEFINITIONS = {
   integromat: integromat.definition,
 }
 
-// don't add the bash script/definitions unless in self host
-// the fact this isn't included in any definitions means it cannot be
-// ran at all
-if (env.SELF_HOSTED) {
-  const bash = require("./steps/bash")
-  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
-  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
-}
-
 /* istanbul ignore next */
 exports.getAction = async function (actionName) {
   if (ACTION_IMPLS[actionName] != null) {

package/src/automations/automationUtils.js

@@ -1,4 +1,9 @@
-const { getTable } = require("../api/controllers/table/utils")
+const CouchDB = require("../db")
+const {
+  isExternalTable,
+  breakExternalTableId,
+} = require("../integrations/utils")
+const { getExternalTable } = require("../api/controllers/table/utils")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -16,7 +21,7 @@ const { getTable } = require("../api/controllers/table/utils")
  * @returns {object} The inputs object which has had all the various types supported by this function converted to their
  * primitive types.
  */
-exports.cleanInputValues = (inputs, schema) => {
+module.exports.cleanInputValues = (inputs, schema) => {
   if (schema == null) {
     return inputs
   }
@@ -58,11 +63,30 @@ exports.cleanInputValues = (inputs, schema) => {
  * @param {object} row The input row structure which requires clean-up after having been through template statements.
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
-exports.cleanUpRow = async (appId, tableId, row) => {
-  let table = await getTable(appId, tableId)
-  return exports.cleanInputValues(row, { properties: table.schema })
+module.exports.cleanUpRow = async (appId, tableId, row) => {
+  const db = new CouchDB(appId)
+  let table
+  if (isExternalTable(tableId)) {
+    const { datasourceId, tableName } = breakExternalTableId(tableId)
+    table = await getExternalTable(appId, datasourceId, tableName)
+  } else {
+    table = await db.get(tableId)
+  }
+
+  return module.exports.cleanInputValues(row, { properties: table.schema })
 }
 
-exports.getError = err => {
-  return typeof err !== "string" ? err.toString() : err
+/**
+ * A utility function for the cleanUpRow, which can be used if only the row ID is known (not the table ID) to clean
+ * up a row after template statements have been replaced. This is specifically useful for the update row action.
+ *
+ * @param {string} appId The instance which the Table/Table is contained under.
+ * @param {string} rowId The ID of the row from which the tableId will be extracted, to get the Table/Table schema.
+ * @param {object} row The input row structure which requires clean-up after having been through template statements.
+ * @returns {Promise<Object>} The cleaned up rows object, which will now have all the required primitive types.
+ */
+module.exports.cleanUpRowById = async (appId, rowId, row) => {
+  const db = new CouchDB(appId)
+  const foundRow = await db.get(rowId)
+  return module.exports.cleanUpRow(appId, foundRow.tableId, row)
 }

package/src/automations/steps/bash.js

@@ -1,6 +1,5 @@
 const { execSync } = require("child_process")
 const { processStringSync } = require("@budibase/string-templates")
-const automationUtils = require("../automationUtils")
 
 exports.definition = {
   name: "Bash Scripting",
@@ -64,7 +63,7 @@ exports.run = async function ({ inputs, context }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/createRow.js

@@ -97,7 +97,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/deleteRow.js

@@ -2,7 +2,6 @@ const rowController = require("../../api/controllers/row")
 const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
-const automationUtils = require("../automationUtils")
 
 exports.definition = {
   description: "Delete a row from your database",
@@ -86,7 +85,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/executeQuery.js

@@ -1,6 +1,5 @@
 const queryController = require("../../api/controllers/query")
 const { buildCtx } = require("./utils")
-const automationUtils = require("../automationUtils")
 
 exports.definition = {
   name: "External Data Connector",
@@ -75,7 +74,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/executeScript.js

@@ -1,6 +1,5 @@
 const scriptController = require("../../api/controllers/script")
 const { buildCtx } = require("./utils")
-const automationUtils = require("../automationUtils")
 
 exports.definition = {
   name: "JS Scripting",
@@ -64,7 +63,7 @@ exports.run = async function ({ inputs, appId, context, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/outgoingWebhook.js

@@ -1,6 +1,5 @@
 const fetch = require("node-fetch")
 const { getFetchResponse } = require("./utils")
-const automationUtils = require("../automationUtils")
 
 const RequestType = {
   POST: "POST",
@@ -128,7 +127,7 @@ exports.run = async function ({ inputs }) {
     /* istanbul ignore next */
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/queryRows.js

@@ -2,7 +2,6 @@ const rowController = require("../../api/controllers/row")
 const tableController = require("../../api/controllers/table")
 const { FieldTypes } = require("../../constants")
 const { buildCtx } = require("./utils")
-const automationUtils = require("../automationUtils")
 
 const SortOrders = {
   ASCENDING: "ascending",
@@ -111,7 +110,7 @@ exports.run = async function ({ inputs, appId }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/sendSmtpEmail.js

@@ -1,5 +1,4 @@
 const { sendSmtpEmail } = require("../../utilities/workerRequests")
-const automationUtils = require("../automationUtils")
 
 exports.definition = {
   description: "Send an email using SMTP",
@@ -62,7 +61,7 @@ exports.run = async function ({ inputs }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/steps/updateRow.js

@@ -64,7 +64,6 @@ exports.run = async function ({ inputs, appId, emitter }) {
       },
     }
   }
-  const tableId = inputs.row.tableId
 
   // clear any falsy properties so that they aren't updated
   for (let propKey of Object.keys(inputs.row)) {
@@ -81,14 +80,15 @@ exports.run = async function ({ inputs, appId, emitter }) {
     },
     params: {
       rowId: inputs.rowId,
-      tableId: tableId,
     },
   })
 
   try {
-    if (tableId) {
-      inputs.row = await automationUtils.cleanUpRow(appId, tableId, inputs.row)
-    }
+    inputs.row = await automationUtils.cleanUpRowById(
+      appId,
+      inputs.rowId,
+      inputs.row
+    )
     await rowController.patch(ctx)
     return {
       row: ctx.body,
@@ -100,7 +100,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: err,
     }
   }
 }

package/src/automations/thread.js

@@ -10,7 +10,6 @@ const env = require("../environment")
 const usage = require("../utilities/usageQuota")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
-const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -69,13 +68,7 @@ class Orchestrator {
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
-    let stopped = false
     for (let step of automation.definition.steps) {
-      // execution stopped, record state for that
-      if (stopped) {
-        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
-        continue
-      }
       let stepFn = await this.getStepFunctionality(step.stepId)
       step.inputs = await processObject(step.inputs, this._context)
       step.inputs = automationUtils.cleanInputValues(
@@ -93,17 +86,10 @@ class Orchestrator {
             context: this._context,
           })
         })
-        this._context.steps.push(outputs)
-        // if filter causes us to stop execution don't break the loop, set a var
-        // so that we can finish iterating through the steps and record that it stopped
         if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          stopped = true
-          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-            ...outputs,
-            ...STOPPED_STATUS,
-          })
-          continue
+          break
         }
+        this._context.steps.push(outputs)
         this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       } catch (err) {
         console.error(`Automation error - ${step.stepId} - ${err}`)
@@ -113,7 +99,7 @@ class Orchestrator {
 
     // Increment quota for automation runs
     if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
-      await usage.update(usage.Properties.AUTOMATION, 1)
+      usage.update(usage.Properties.AUTOMATION, 1)
     }
     return this.executionOutput
   }

package/src/constants/index.js

@@ -42,7 +42,6 @@ exports.FieldTypes = {
   FORMULA: "formula",
   AUTO: "auto",
   JSON: "json",
-  INTERNAL: "internal",
 }
 
 exports.RelationshipTypes = {

package/src/integrations/base/sql.ts

@@ -98,9 +98,7 @@ function addFilters(
 }
 
 function addRelationships(
-  knex: Knex,
   query: KnexQuery,
-  fields: string | string[],
   fromTable: string,
   relationships: RelationshipsJson[] | undefined
 ): KnexQuery {
@@ -116,7 +114,7 @@ function addRelationships(
       query = query.leftJoin(
         toTable,
         `${fromTable}.${from}`,
-        `${toTable}.${to}`
+        `${relationship.tableName}.${to}`
       )
     } else {
       const throughTable = relationship.through
@@ -132,7 +130,7 @@ function addRelationships(
         .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
     }
   }
-  return query.limit(BASE_LIMIT)
+  return query
 }
 
 function buildCreate(
@@ -201,7 +199,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
     [tableName]: query,
   }).select(selectStatement)
   // handle joins
-  return addRelationships(knex, preQuery, selectStatement, tableName, relationships)
+  return addRelationships(preQuery, tableName, relationships)
 }
 
 function buildUpdate(

package/src/integrations/tests/mongo.spec.js

@@ -27,7 +27,7 @@ describe("MongoDB Integration", () => {
     const body = {
       name: "Hello"
     }
-    await config.integration.create({
+    const response = await config.integration.create({ 
       index: indexName,
       json: body,
       extra: { collection: 'testCollection', actionTypes: 'insertOne'}
@@ -54,7 +54,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    await config.integration.delete(query)
+    const response = await config.integration.delete(query)
     expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -65,7 +65,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'updateOne'}
     }
-    await config.integration.update(query)
+    const response = await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -75,14 +75,10 @@ describe("MongoDB Integration", () => {
     const query = {
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
+    // Weird, need to do an IIFE for jest to recognize that it throws
+    expect(() => config.integration.read(query)()).toThrow(expect.any(Object))
 
-    let error = null
-    try {
-      await config.integration.read(query)
-    } catch (err) {
-      error = err
-    }
-    expect(error).toBeDefined()
     restore()
   })
+
 })

package/src/integrations/utils.ts

@@ -5,7 +5,6 @@ const { DocumentTypes, SEPARATOR } = require("../db/utils")
 const { FieldTypes } = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
-const ROW_ID_REGEX = /^\[.*]$/g
 
 export function isExternalTable(tableId: string) {
   return tableId.includes(DocumentTypes.DATASOURCE)
@@ -33,20 +32,6 @@ export function generateRowIdField(keyProps: any[] = []) {
   return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
 }
 
-export function isRowId(field: any) {
-  return Array.isArray(field) || (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
-}
-
-export function convertRowId(field: any) {
-  if (Array.isArray(field)) {
-    return field[0]
-  }
-  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
-    return field.substring(1, field.length - 1)
-  }
-  return field
-}
-
 // should always return an array
 export function breakRowIdField(_id: string | { _id: string }): any[] {
   if (!_id) {

package/src/middleware/currentapp.js

@@ -45,7 +45,7 @@ module.exports = async (ctx, next) => {
     const globalUser = await getCachedSelf(ctx, requestAppId)
     appId = requestAppId
     // retrieving global user gets the right role
-    roleId = globalUser.roleId || roleId
+    roleId = globalUser.roleId || BUILTIN_ROLE_IDS.BASIC
   }
 
   // nothing more to do

package/src/middleware/tests/currentapp.spec.js

@@ -127,8 +127,8 @@ describe("Current app middleware", () => {
       } else {
         expect(cookieFn).not.toHaveBeenCalled()
       }
-      expect(config.ctx.roleId).toEqual("PUBLIC")
-      expect(config.ctx.user.role._id).toEqual("PUBLIC")
+      expect(config.ctx.roleId).toEqual("BASIC")
+      expect(config.ctx.user.role._id).toEqual("BASIC")
       expect(config.ctx.appId).toEqual("app_test")
       expect(config.next).toHaveBeenCalled()
     }
@@ -163,7 +163,7 @@ describe("Current app middleware", () => {
             return "app_test"
           },
           setCookie: jest.fn(),
-          getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
+          getCookie: () => ({appId: "app_test", roleId: "BASIC"}),
         },
         constants: { Cookies: {} },
       }))

package/src/utilities/fileSystem/index.js

@@ -106,86 +106,64 @@ exports.apiFileReturn = contents => {
 }
 
 /**
- * Local utility to back up the database state for an app, excluding global user
- * data or user relationships.
- * @param {string} appId The app to backup
- * @param {object} config Config to send to export DB
- * @returns {*} either a string or a stream of the backup
+ * Takes a copy of the database state for an app to the object store.
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {string} backupName The name of the backup located in the object store.
+ * @return The backup has been completed when this promise completes and returns a file stream
+ * to the temporary backup file (to return via API if required).
  */
-const backupAppData = async (appId, config) => {
-  return await exports.exportDB(appId, {
-    ...config,
+exports.performBackup = async (appId, backupName) => {
+  return exports.exportDB(appId, {
+    exportName: backupName,
     filter: doc =>
       !(
         doc._id.includes(USER_METDATA_PREFIX) ||
-        doc._id.includes(LINK_USER_METADATA_PREFIX)
+        doc.includes(LINK_USER_METADATA_PREFIX)
       ),
   })
 }
 
 /**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
- */
-exports.performBackup = async (appId, backupName) => {
-  return await backupAppData(appId, { exportName: backupName })
-}
-
-/**
- * Streams a backup of the database state for an app
- * @param {string} appId The ID of the app which is to be backed up.
- * @returns {*} a readable stream of the backup which is written in real time
- */
-exports.streamBackup = async appId => {
-  return await backupAppData(appId, { stream: true })
-}
-
-/**
- * Exports a DB to either file or a variable (memory).
+ * exports a DB to either file or a variable (memory).
  * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - provide a filename to write the backup to a file
- * @param {boolean} stream optional - whether to perform a full backup
+ * @param {string} exportName optional - the file name to export to, if not in memory.
  * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return {*} either a readable stream or a string
+ * @return Either the file stream or the variable (if no export name provided).
  */
-exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
-  const instanceDb = new CouchDB(dbName)
-
-  // Stream the dump if required
-  if (stream) {
-    const memStream = new MemoryStream()
-    instanceDb.dump(memStream, { filter })
-    return memStream
-  }
-
-  // Write the dump to file if required
+exports.exportDB = async (
+  dbName,
+  { exportName, filter } = { exportName: undefined, filter: undefined }
+) => {
+  let stream,
+    appString = "",
+    path = null
   if (exportName) {
-    const path = join(budibaseTempDir(), exportName)
-    const writeStream = fs.createWriteStream(path)
-    await instanceDb.dump(writeStream, { filter })
-
-    // Upload the dump to the object store if self hosted
-    if (env.SELF_HOSTED) {
-      await streamUpload(
-        ObjectStoreBuckets.BACKUPS,
-        join(dbName, exportName),
-        fs.createReadStream(path)
-      )
-    }
-
-    return fs.createReadStream(path)
+    path = join(budibaseTempDir(), exportName)
+    stream = fs.createWriteStream(path)
+  } else {
+    stream = new MemoryStream()
+    stream.on("data", chunk => {
+      appString += chunk.toString()
+    })
   }
-
-  // Stringify the dump in memory if required
-  const memStream = new MemoryStream()
-  let appString = ""
-  memStream.on("data", chunk => {
-    appString += chunk.toString()
+  // perform couch dump
+  const instanceDb = new CouchDB(dbName)
+  await instanceDb.dump(stream, {
+    filter,
   })
-  await instanceDb.dump(memStream, { filter })
-  return appString
+  // just in memory, return the final string
+  if (!exportName) {
+    return appString
+  }
+  // write the file to the object store
+  if (env.SELF_HOSTED) {
+    await streamUpload(
+      ObjectStoreBuckets.BACKUPS,
+      join(dbName, exportName),
+      fs.createReadStream(path)
+    )
+  }
+  return fs.createReadStream(path)
 }
 
 /**

package/src/utilities/global.js

@@ -26,7 +26,7 @@ exports.updateAppRole = (appId, user) => {
   if (!user.roleId && user.builder && user.builder.global) {
     user.roleId = BUILTIN_ROLE_IDS.ADMIN
   } else if (!user.roleId) {
-    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    user.roleId = BUILTIN_ROLE_IDS.BASIC
   }
   delete user.roles
   return user