@budibase/server 0.9.171 → 0.9.173-alpha.2

package/src/api/controllers/row/ExternalRequest.ts

@@ -15,8 +15,9 @@ import {
 import {
   breakRowIdField,
   generateRowIdField,
+  isRowId,
+  convertRowId,
 } from "../../../integrations/utils"
-import { RelationshipTypes } from "../../../constants"
 
 interface ManyRelationship {
   tableId?: string
@@ -36,7 +37,11 @@ interface RunConfig {
 
 module External {
   const { makeExternalQuery } = require("./utils")
-  const { DataSourceOperation, FieldTypes } = require("../../../constants")
+  const {
+    DataSourceOperation,
+    FieldTypes,
+    RelationshipTypes,
+  } = require("../../../constants")
   const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
   const { processObjectSync } = require("@budibase/string-templates")
   const { cloneDeep } = require("lodash/fp")
@@ -83,6 +88,48 @@ module External {
     }
   }
 
+  /**
+   * This function checks the incoming parameters to make sure all the inputs are
+   * valid based on on the table schema. The main thing this is looking for is when a
+   * user has made use of the _id field of a row for a foreign key or a search parameter.
+   * In these cases the key will be sent up as [1], rather than 1. In these cases we will
+   * simplify it down to the requirements. This function is quite complex as we try to be
+   * relatively restrictive over what types of columns we will perform this action for.
+   */
+  function cleanupConfig(config: RunConfig, table: Table): RunConfig {
+    const primaryOptions = [
+      FieldTypes.STRING,
+      FieldTypes.LONGFORM,
+      FieldTypes.OPTIONS,
+      FieldTypes.NUMBER,
+    ]
+    // filter out fields which cannot be keys
+    const fieldNames = Object.entries(table.schema)
+      .filter(schema => primaryOptions.find(val => val === schema[1].type))
+      .map(([fieldName]) => fieldName)
+    const iterateObject = (obj: { [key: string]: any }) => {
+      for (let [field, value] of Object.entries(obj)) {
+        if (fieldNames.find(name => name === field) && isRowId(value)) {
+          obj[field] = convertRowId(value)
+        }
+      }
+    }
+    // check the row and filters to make sure they aren't a key of some sort
+    if (config.filters) {
+      for (let filter of Object.values(config.filters)) {
+        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
+          continue
+        }
+        iterateObject(filter)
+      }
+    }
+    if (config.row) {
+      iterateObject(config.row)
+    }
+
+    return config
+  }
+
   function generateIdForRow(row: Row | undefined, table: Table): string {
     const primary = table.primary
     if (!row || !primary) {
@@ -509,7 +556,7 @@ module External {
       return fields
     }
 
-    async run({ id, row, filters, sort, paginate }: RunConfig) {
+    async run(config: RunConfig) {
       const { appId, operation, tableId } = this
       let { datasourceId, tableName } = breakExternalTableId(tableId)
       if (!this.datasource) {
@@ -525,9 +572,11 @@ module External {
       if (!table) {
         throw `Unable to process query, table "${tableName}" not defined.`
       }
-      // clean up row on ingress using schema
+      // look for specific components of config which may not be considered acceptable
+      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
       filters = buildFilters(id, filters || {}, table)
       const relationships = this.buildRelationships(table)
+      // clean up row on ingress using schema
       const processed = this.inputProcessing(row, table)
       row = processed.row
       if (

package/src/api/controllers/script.js

@@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
-
-  ctx.body = executor.execute()
+  const { script, context } = ctx.request.body
+  const runner = new ScriptRunner(script, context)
+  ctx.body = runner.execute()
 }
 
 exports.save = async function (ctx) {

package/src/api/controllers/table/index.js

@@ -9,11 +9,7 @@ const {
   BudibaseInternalDB,
 } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions, getExternalTable } = require("./utils")
-const {
-  isExternalTable,
-  breakExternalTableId,
-} = require("../../../integrations/utils")
+const { TableSaveFunctions, getTable } = require("./utils")
 
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
@@ -48,14 +44,8 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
   const tableId = ctx.params.id
-  if (isExternalTable(tableId)) {
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    ctx.body = await getExternalTable(ctx.appId, datasourceId, tableName)
-  } else {
-    ctx.body = await db.get(ctx.params.id)
-  }
+  ctx.body = await getTable(ctx.appId, tableId)
 }
 
 exports.save = async function (ctx) {
@@ -91,6 +81,9 @@ exports.save = async function (ctx) {
     for (let propKey of Object.keys(tableToSave.schema)) {
       let column = tableToSave.schema[propKey]
       let oldColumn = oldTable.schema[propKey]
+      if (oldColumn && oldColumn.type === "internal") {
+        oldColumn.type = "auto"
+      }
       if (oldColumn && oldColumn.type !== column.type) {
         ctx.throw(400, "Cannot change the type of a column")
       }

package/src/api/controllers/table/utils.js

@@ -1,5 +1,4 @@
 const CouchDB = require("../../../db")
-const linkRows = require("../../../db/linkedRows")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
@@ -10,6 +9,12 @@ const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
 const { inputProcessing } = require("../../../utilities/rowProcessor")
 const { USERS_TABLE_SCHEMA } = require("../../../constants")
+const {
+  isExternalTable,
+  breakExternalTableId,
+} = require("../../../integrations/utils")
+const { getViews, saveView } = require("../view/utils")
+const viewTemplate = require("../view/viewBuilder")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -22,6 +27,7 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   }
   // check for renaming of columns or deleted columns
   if (rename || deletedColumns.length !== 0) {
+    // Update all rows
     const rows = await db.allDocs(
       getRowParams(updatedTable._id, null, {
         include_docs: true,
@@ -36,6 +42,9 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
       }
       return doc
     })
+
+    // Update views
+    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -93,19 +102,10 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
         }
       }
 
-      // make sure link rows are up to date
-      finalData.push(
-        linkRows.updateLinks({
-          appId,
-          eventType: linkRows.EventType.ROW_SAVE,
-          row,
-          tableId: row.tableId,
-          table,
-        })
-      )
+      finalData.push(row)
     }
 
-    await db.bulkDocs(await Promise.all(finalData))
+    await db.bulkDocs(finalData)
     let response = await db.put(table)
     table._rev = response._rev
   }
@@ -233,4 +233,86 @@ exports.getExternalTable = async (appId, datasourceId, tableName) => {
   return entities[tableName]
 }
 
+exports.getTable = async (appId, tableId) => {
+  const db = new CouchDB(appId)
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    return exports.getExternalTable(appId, datasourceId, tableName)
+  } else {
+    return db.get(tableId)
+  }
+}
+
+exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
+  const views = await getViews(db)
+  const tableViews = views.filter(view => view.meta.tableId === table._id)
+
+  // Check each table view to see if impacted by this table action
+  for (let view of tableViews) {
+    let needsUpdated = false
+
+    // First check for renames, otherwise check for deletions
+    if (rename) {
+      // Update calculation field if required
+      if (view.meta.field === rename.old) {
+        view.meta.field = rename.updated
+        needsUpdated = true
+      }
+
+      // Update group by field if required
+      if (view.meta.groupBy === rename.old) {
+        view.meta.groupBy = rename.updated
+        needsUpdated = true
+      }
+
+      // Update filters if required
+      if (view.meta.filters) {
+        view.meta.filters.forEach(filter => {
+          if (filter.key === rename.old) {
+            filter.key = rename.updated
+            needsUpdated = true
+          }
+        })
+      }
+    } else if (deletedColumns) {
+      deletedColumns.forEach(column => {
+        // Remove calculation statement if required
+        if (view.meta.field === column) {
+          delete view.meta.field
+          delete view.meta.calculation
+          delete view.meta.groupBy
+          needsUpdated = true
+        }
+
+        // Remove group by field if required
+        if (view.meta.groupBy === column) {
+          delete view.meta.groupBy
+          needsUpdated = true
+        }
+
+        // Remove filters referencing deleted field if required
+        if (view.meta.filters && view.meta.filters.length) {
+          const initialLength = view.meta.filters.length
+          view.meta.filters = view.meta.filters.filter(filter => {
+            return filter.key !== column
+          })
+          if (initialLength !== view.meta.filters.length) {
+            needsUpdated = true
+          }
+        }
+      })
+    }
+
+    // Update view if required
+    if (needsUpdated) {
+      const newViewTemplate = viewTemplate(view.meta)
+      await saveView(db, null, view.name, newViewTemplate)
+      if (!newViewTemplate.meta.schema) {
+        newViewTemplate.meta.schema = table.schema
+      }
+      table.views[view.name] = newViewTemplate.meta
+    }
+  }
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

package/src/api/controllers/templates.js

@@ -7,11 +7,23 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function (ctx) {
   const { type = "app" } = ctx.query
-  const response = await fetch(
-    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-  )
-  const json = await response.json()
-  ctx.body = Object.values(json.templates[type])
+  let response,
+    error = false
+  try {
+    response = await fetch(`https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`)
+    if (response.status !== 200) {
+      error = true
+    }
+  } catch (err) {
+    error = true
+  }
+  // if there is an error, simply return no templates
+  if (!error && response) {
+    const json = await response.json()
+    ctx.body = Object.values(json.templates[type])
+  } else {
+    ctx.body = []
+  }
 }
 
 // can't currently test this, have to ignore from coverage

package/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap

@@ -44,7 +44,7 @@ Object {
 exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
 Object {
   "map": "function (doc) {
-      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && doc[\\"Name\\"] === \\"Test\\" || doc[\\"Yes\\"] > \\"Value\\") {
+      if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && (doc[\\"Name\\"] === \\"Test\\" || doc[\\"Yes\\"] > \\"Value\\")) {
         emit(doc[\\"_id\\"], doc[\\"undefined\\"]);
       }
     }",

package/src/api/controllers/view/viewBuilder.js

@@ -10,6 +10,15 @@ const TOKEN_MAP = {
   OR: "||",
 }
 
+const isEmptyExpression = key => {
+  return `(
+      doc["${key}"] === undefined ||
+      doc["${key}"] === null ||
+      doc["${key}"] === "" ||
+      (Array.isArray(doc["${key}"]) && doc["${key}"].length === 0)
+  )`
+}
+
 const GROUP_PROPERTY = {
   group: {
     type: "string",
@@ -72,6 +81,10 @@ function parseFilterExpression(filters) {
       expression.push(
         `doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
       )
+    } else if (filter.condition === "EMPTY") {
+      expression.push(isEmptyExpression(filter.key))
+    } else if (filter.condition === "NOT_EMPTY") {
+      expression.push(`!${isEmptyExpression(filter.key)}`)
     } else {
       const value =
         typeof filter.value == "string" ? `"${filter.value}"` : filter.value
@@ -113,7 +126,7 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
     delete filters[0].conjunction
   }
   const parsedFilters = parseFilterExpression(filters)
-  const filterExpression = parsedFilters ? `&& ${parsedFilters}` : ""
+  const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
 
   const emitExpression = parseEmitExpression(field, groupBy)
 

package/src/api/routes/application.js

@@ -7,6 +7,7 @@ const usage = require("../../middleware/usageQuota")
 const router = Router()
 
 router
+  .post("/api/applications/:appId/sync", authorized(BUILDER), controller.sync)
   .post("/api/applications", authorized(BUILDER), usage, controller.create)
   .get("/api/applications/:appId/definition", controller.fetchAppDefinition)
   .get("/api/applications", controller.fetch)

package/src/api/routes/dev.js

@@ -6,11 +6,16 @@ const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
-if (env.isDev() || env.isTest()) {
+function redirectPath(path) {
   router
-    .get("/api/global/:devPath(.*)", controller.redirectGet)
-    .post("/api/global/:devPath(.*)", controller.redirectPost)
-    .delete("/api/global/:devPath(.*)", controller.redirectDelete)
+    .get(`/api/${path}/:devPath(.*)`, controller.buildRedirectGet(path))
+    .post(`/api/${path}/:devPath(.*)`, controller.buildRedirectPost(path))
+    .delete(`/api/${path}/:devPath(.*)`, controller.buildRedirectDelete(path))
+}
+
+if (env.isDev() || env.isTest()) {
+  redirectPath("global")
+  redirectPath("system")
 }
 
 router

package/src/api/routes/query.js

@@ -31,7 +31,8 @@ function generateQueryValidation() {
     })),
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true)
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
   }))
 }
 
@@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
     parameters: Joi.object({}).required().unknown(true)
   }))
 }

package/src/api/routes/tests/auth.spec.js

@@ -13,8 +13,7 @@ describe("/authenticate", () => {
 
   describe("fetch self", () => {
     it("should be able to fetch self", async () => {
-      await config.createUser("test@test.com", "p4ssw0rd")
-      const headers = await config.login("test@test.com", "p4ssw0rd", { userId: "us_uuid1" })
+      const headers = await config.login()
       const res = await request
         .get(`/api/self`)
         .set(headers)

package/src/api/routes/tests/permissions.spec.js

@@ -89,6 +89,9 @@ describe("/permission", () => {
 
   describe("check public user allowed", () => {
     it("should be able to read the row", async () => {
+      // replicate changes before checking permissions
+      await config.deploy()
+
       const res = await request
         .get(`/api/${table._id}/rows`)
         .set(config.publicHeaders())

package/src/api/routes/tests/query.spec.js

@@ -94,9 +94,9 @@ describe("/queries", () => {
         const query = await config.createQuery()
         const res = await request
           .get(`/api/queries/${query._id}`)
-          .set(await config.roleHeaders({}))
-          .expect("Content-Type", /json/)
+          .set(await config.defaultHeaders())
           .expect(200)
+          .expect("Content-Type", /json/)
         expect(res.body.fields).toBeUndefined()
         expect(res.body.parameters).toBeUndefined()
         expect(res.body.schema).toBeUndefined()

package/src/api/routes/tests/routing.spec.js

@@ -21,16 +21,31 @@ describe("/routing", () => {
     screen2.routing.roleId = BUILTIN_ROLE_IDS.POWER
     screen2.routing.route = route
     screen2 = await config.createScreen(screen2)
+    await config.deploy()
   })
 
   describe("fetch", () => {
+    it("prevents a public user from accessing development app", async () => {
+      await request
+        .get(`/api/routing/client`)
+        .set(config.publicHeaders({ prodApp: false }))
+        .expect(302)
+    })
+
+    it("prevents a non builder from accessing development app", async () => {
+      await request
+        .get(`/api/routing/client`)
+        .set(await config.roleHeaders({
+          roleId: BUILTIN_ROLE_IDS.BASIC,
+          prodApp: false
+        }))
+        .expect(302)
+    })
     it("returns the correct routing for basic user", async () => {
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({
-          email: "basic@test.com",
-          roleId: BUILTIN_ROLE_IDS.BASIC,
-          builder: false
+          roleId: BUILTIN_ROLE_IDS.BASIC
         }))
         .expect("Content-Type", /json/)
         .expect(200)
@@ -49,9 +64,7 @@ describe("/routing", () => {
       const res = await request
         .get(`/api/routing/client`)
         .set(await config.roleHeaders({
-          email: "basic@test.com",
-          roleId: BUILTIN_ROLE_IDS.POWER,
-          builder: false,
+          roleId: BUILTIN_ROLE_IDS.POWER
         }))
         .expect("Content-Type", /json/)
         .expect(200)
@@ -71,9 +84,12 @@ describe("/routing", () => {
     it("should fetch all routes for builder", async () => {
       const res = await request
         .get(`/api/routing`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
+        .set(await config.roleHeaders({
+          roleId: BUILTIN_ROLE_IDS.POWER,
+          builder: true,
+        }))
         .expect(200)
+        .expect("Content-Type", /json/)
       expect(res.body.routes).toBeDefined()
       expect(res.body.routes[route].subpaths[route]).toBeDefined()
       const subpath = res.body.routes[route].subpaths[route]

package/src/api/routes/tests/user.spec.js

@@ -38,7 +38,7 @@ describe("/users", () => {
     })
 
     it("should apply authorization to endpoint", async () => {
-      await config.createUser("brenda@brenda.com", "brendas_password")
+      await config.createUser()
       await checkPermissionsEndpoint({
         config,
         request,

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -50,9 +50,10 @@ exports.createRequest = (request, method, url, body) => {
 }
 
 exports.checkBuilderEndpoint = async ({ config, method, url, body }) => {
-  const headers = await config.login("test@test.com", "test", {
+  const headers = await config.login({
     userId: "us_fail",
     builder: false,
+    prodApp: true,
   })
   await exports
     .createRequest(config.request, method, url, body)
@@ -68,11 +69,9 @@ exports.checkPermissionsEndpoint = async ({
   passRole,
   failRole,
 }) => {
-  const password = "PASSWORD"
-  let user = await config.createUser("pass@budibase.com", password, passRole)
-  const passHeader = await config.login("pass@budibase.com", password, {
+  const passHeader = await config.login({
     roleId: passRole,
-    userId: user.globalId,
+    prodApp: true,
   })
 
   await exports
@@ -82,13 +81,12 @@ exports.checkPermissionsEndpoint = async ({
 
   let failHeader
   if (failRole === BUILTIN_ROLE_IDS.PUBLIC) {
-    failHeader = config.publicHeaders()
+    failHeader = config.publicHeaders({ prodApp: true })
   } else {
-    user = await config.createUser("fail@budibase.com", password, failRole)
-    failHeader = await config.login("fail@budibase.com", password, {
+    failHeader = await config.login({
       roleId: failRole,
-      userId: user.globalId,
       builder: false,
+      prodApp: true,
     })
   }
 

package/src/api/routes/tests/webhook.spec.js

@@ -112,8 +112,11 @@ describe("/webhooks", () => {
 
   describe("trigger", () => {
     it("should allow triggering from public", async () => {
+      // replicate changes before checking webhook
+      await config.deploy()
+
       const res = await request
-        .post(`/api/webhooks/trigger/${config.getAppId()}/${webhook._id}`)
+        .post(`/api/webhooks/trigger/${config.prodAppId}/${webhook._id}`)
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.message).toBeDefined()

package/src/automations/actions.js

@@ -3,7 +3,6 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const executeScript = require("./steps/executeScript")
-const bash = require("./steps/bash")
 const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const serverLog = require("./steps/serverLog")
@@ -14,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+const env = require("../environment")
 
 const ACTION_IMPLS = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
@@ -22,7 +22,6 @@ const ACTION_IMPLS = {
   DELETE_ROW: deleteRow.run,
   OUTGOING_WEBHOOK: outgoingWebhook.run,
   EXECUTE_SCRIPT: executeScript.run,
-  EXECUTE_BASH: bash.run,
   EXECUTE_QUERY: executeQuery.run,
   SERVER_LOG: serverLog.run,
   DELAY: delay.run,
@@ -42,7 +41,6 @@ const ACTION_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
   EXECUTE_SCRIPT: executeScript.definition,
   EXECUTE_QUERY: executeQuery.definition,
-  EXECUTE_BASH: bash.definition,
   SERVER_LOG: serverLog.definition,
   DELAY: delay.definition,
   FILTER: filter.definition,
@@ -54,6 +52,15 @@ const ACTION_DEFINITIONS = {
   integromat: integromat.definition,
 }
 
+// don't add the bash script/definitions unless in self host
+// the fact this isn't included in any definitions means it cannot be
+// ran at all
+if (env.SELF_HOSTED) {
+  const bash = require("./steps/bash")
+  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
+  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
 /* istanbul ignore next */
 exports.getAction = async function (actionName) {
   if (ACTION_IMPLS[actionName] != null) {