@budibase/server 0.9.167-alpha.0 → 0.9.167-alpha.12

package/dist/utilities/global.js

@@ -33,7 +33,7 @@ exports.updateAppRole = (appId, user) => {
         user.roleId = BUILTIN_ROLE_IDS.ADMIN;
     }
     else if (!user.roleId) {
-        user.roleId = BUILTIN_ROLE_IDS.BASIC;
+        user.roleId = BUILTIN_ROLE_IDS.PUBLIC;
     }
     delete user.roles;
     return user;

package/dist/utilities/rowProcessor/index.js

@@ -149,6 +149,10 @@ exports.processAutoColumn = processAutoColumn;
  * @returns {object} The coerced value
  */
 exports.coerce = (row, type) => {
+    // no coercion specified for type, skip it
+    if (!TYPE_TRANSFORM_MAP[type]) {
+        return row;
+    }
     // eslint-disable-next-line no-prototype-builtins
     if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
         return TYPE_TRANSFORM_MAP[type][row];
@@ -190,6 +194,10 @@ exports.inputProcessing = (user = {}, table, row, opts = { noAutoRelationships:
             clonedRow[key] = exports.coerce(value, field.type);
         }
     }
+    if (!clonedRow._id || !clonedRow._rev) {
+        clonedRow._id = row._id;
+        clonedRow._rev = row._rev;
+    }
     // handle auto columns - this returns an object like {table, row}
     return processAutoColumn(user, copiedTable, clonedRow, opts);
 };

package/dist/utilities/scriptRunner.js

@@ -0,0 +1,19 @@
+"use strict";
+const fetch = require("node-fetch");
+const { VM, VMScript } = require("vm2");
+class ScriptRunner {
+    constructor(script, context) {
+        const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
+        this.vm = new VM();
+        this.results = { out: "" };
+        this.vm.setGlobals(context);
+        this.vm.setGlobal("fetch", fetch);
+        this.vm.setGlobal("results", this.results);
+        this.script = new VMScript(code);
+    }
+    execute() {
+        this.vm.run(this.script);
+        return this.results.out;
+    }
+}
+module.exports = ScriptRunner;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.167-alpha.0",
+  "version": "0.9.167-alpha.12",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.167-alpha.0",
-    "@budibase/client": "^0.9.167-alpha.0",
-    "@budibase/string-templates": "^0.9.167-alpha.0",
+    "@budibase/auth": "^0.9.167-alpha.12",
+    "@budibase/client": "^0.9.167-alpha.12",
+    "@budibase/string-templates": "^0.9.167-alpha.12",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -119,6 +119,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -148,5 +149,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "54fff6db18dced835ab577cd85453ef11606b0d0"
+  "gitHead": "eed61105f0b7a018bd4be19f655fc7f1020252be"
 }

package/src/api/controllers/dev.js

@@ -7,11 +7,13 @@ const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
 
-async function redirect(ctx, method) {
+async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
-    checkSlashesInUrl(`${env.WORKER_URL}/api/global/${devPath}?${queryString}`),
+    checkSlashesInUrl(
+      `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
+    ),
     request(
       ctx,
       {
@@ -41,16 +43,22 @@ async function redirect(ctx, method) {
   ctx.cookies
 }
 
-exports.redirectGet = async ctx => {
-  await redirect(ctx, "GET")
+exports.buildRedirectGet = path => {
+  return async ctx => {
+    await redirect(ctx, "GET", path)
+  }
 }
 
-exports.redirectPost = async ctx => {
-  await redirect(ctx, "POST")
+exports.buildRedirectPost = path => {
+  return async ctx => {
+    await redirect(ctx, "POST", path)
+  }
 }
 
-exports.redirectDelete = async ctx => {
-  await redirect(ctx, "DELETE")
+exports.buildRedirectDelete = path => {
+  return async ctx => {
+    await redirect(ctx, "DELETE", path)
+  }
 }
 
 exports.clearLock = async ctx => {

package/src/api/controllers/query.js

@@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -28,12 +29,39 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  if (!Array.isArray(resp)) {
-    resp = [resp]
-  }
   return resp
 }
 
+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb } = ctx.request.body
-
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-
   const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  // get all the potential fields in the schema
-  const keys = rows.flatMap(Object.keys)
+  const { rows, keys } = await runAndTransform(
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )
 
   ctx.body = {
     rows,
@@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-
   const integration = new Integration(datasource.config)
+
   // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
   // cleanup
   if (integration.end) {
     integration.end()

package/src/api/controllers/script.js

@@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
-
-  ctx.body = executor.execute()
+  const { script, context } = ctx.request.body
+  const runner = new ScriptRunner(script, context)
+  ctx.body = runner.execute()
 }
 
 exports.save = async function (ctx) {

package/src/api/controllers/table/index.js

@@ -91,6 +91,9 @@ exports.save = async function (ctx) {
     for (let propKey of Object.keys(tableToSave.schema)) {
       let column = tableToSave.schema[propKey]
       let oldColumn = oldTable.schema[propKey]
+      if (oldColumn && oldColumn.type === "internal") {
+        oldColumn.type = "auto"
+      }
       if (oldColumn && oldColumn.type !== column.type) {
         ctx.throw(400, "Cannot change the type of a column")
       }

package/src/api/controllers/table/utils.js

@@ -1,5 +1,4 @@
 const CouchDB = require("../../../db")
-const linkRows = require("../../../db/linkedRows")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
@@ -93,19 +92,10 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
         }
       }
 
-      // make sure link rows are up to date
-      finalData.push(
-        linkRows.updateLinks({
-          appId,
-          eventType: linkRows.EventType.ROW_SAVE,
-          row,
-          tableId: row.tableId,
-          table,
-        })
-      )
+      finalData.push(row)
     }
 
-    await db.bulkDocs(await Promise.all(finalData))
+    await db.bulkDocs(finalData)
     let response = await db.put(table)
     table._rev = response._rev
   }

package/src/api/controllers/templates.js

@@ -7,11 +7,23 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function (ctx) {
   const { type = "app" } = ctx.query
-  const response = await fetch(
-    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
-  )
-  const json = await response.json()
-  ctx.body = Object.values(json.templates[type])
+  let response,
+    error = false
+  try {
+    response = await fetch(`https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`)
+    if (response.status !== 200) {
+      error = true
+    }
+  } catch (err) {
+    error = true
+  }
+  // if there is an error, simply return no templates
+  if (!error && response) {
+    const json = await response.json()
+    ctx.body = Object.values(json.templates[type])
+  } else {
+    ctx.body = []
+  }
 }
 
 // can't currently test this, have to ignore from coverage

package/src/api/routes/dev.js

@@ -6,11 +6,16 @@ const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
-if (env.isDev() || env.isTest()) {
+function redirectPath(path) {
   router
-    .get("/api/global/:devPath(.*)", controller.redirectGet)
-    .post("/api/global/:devPath(.*)", controller.redirectPost)
-    .delete("/api/global/:devPath(.*)", controller.redirectDelete)
+    .get(`/api/${path}/:devPath(.*)`, controller.buildRedirectGet(path))
+    .post(`/api/${path}/:devPath(.*)`, controller.buildRedirectPost(path))
+    .delete(`/api/${path}/:devPath(.*)`, controller.buildRedirectDelete(path))
+}
+
+if (env.isDev() || env.isTest()) {
+  redirectPath("global")
+  redirectPath("system")
 }
 
 router

package/src/api/routes/query.js

@@ -31,7 +31,8 @@ function generateQueryValidation() {
     })),
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true)
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
   }))
 }
 
@@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
     parameters: Joi.object({}).required().unknown(true)
   }))
 }

package/src/automations/actions.js

@@ -3,7 +3,6 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const executeScript = require("./steps/executeScript")
-const bash = require("./steps/bash")
 const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const serverLog = require("./steps/serverLog")
@@ -14,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+const env = require("../environment")
 
 const ACTION_IMPLS = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
@@ -22,7 +22,6 @@ const ACTION_IMPLS = {
   DELETE_ROW: deleteRow.run,
   OUTGOING_WEBHOOK: outgoingWebhook.run,
   EXECUTE_SCRIPT: executeScript.run,
-  EXECUTE_BASH: bash.run,
   EXECUTE_QUERY: executeQuery.run,
   SERVER_LOG: serverLog.run,
   DELAY: delay.run,
@@ -42,7 +41,6 @@ const ACTION_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
   EXECUTE_SCRIPT: executeScript.definition,
   EXECUTE_QUERY: executeQuery.definition,
-  EXECUTE_BASH: bash.definition,
   SERVER_LOG: serverLog.definition,
   DELAY: delay.definition,
   FILTER: filter.definition,
@@ -54,6 +52,15 @@ const ACTION_DEFINITIONS = {
   integromat: integromat.definition,
 }
 
+// don't add the bash script/definitions unless in self host
+// the fact this isn't included in any definitions means it cannot be
+// ran at all
+if (env.SELF_HOSTED) {
+  const bash = require("./steps/bash")
+  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
+  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
 /* istanbul ignore next */
 exports.getAction = async function (actionName) {
   if (ACTION_IMPLS[actionName] != null) {

package/src/automations/thread.js

@@ -10,6 +10,7 @@ const env = require("../environment")
 const usage = require("../utilities/usageQuota")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -68,7 +69,13 @@ class Orchestrator {
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
+    let stopped = false
     for (let step of automation.definition.steps) {
+      // execution stopped, record state for that
+      if (stopped) {
+        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+        continue
+      }
       let stepFn = await this.getStepFunctionality(step.stepId)
       step.inputs = await processObject(step.inputs, this._context)
       step.inputs = automationUtils.cleanInputValues(
@@ -86,10 +93,17 @@ class Orchestrator {
             context: this._context,
           })
         })
+        this._context.steps.push(outputs)
+        // if filter causes us to stop execution don't break the loop, set a var
+        // so that we can finish iterating through the steps and record that it stopped
         if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          break
+          stopped = true
+          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+            ...outputs,
+            ...STOPPED_STATUS,
+          })
+          continue
         }
-        this._context.steps.push(outputs)
         this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       } catch (err) {
         console.error(`Automation error - ${step.stepId} - ${err}`)
@@ -99,7 +113,7 @@ class Orchestrator {
 
     // Increment quota for automation runs
     if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
-      usage.update(usage.Properties.AUTOMATION, 1)
+      await usage.update(usage.Properties.AUTOMATION, 1)
     }
     return this.executionOutput
   }

package/src/constants/index.js

@@ -42,6 +42,7 @@ exports.FieldTypes = {
   FORMULA: "formula",
   AUTO: "auto",
   JSON: "json",
+  INTERNAL: "internal",
 }
 
 exports.RelationshipTypes = {

package/src/integrations/base/sql.ts

@@ -98,7 +98,9 @@ function addFilters(
 }
 
 function addRelationships(
+  knex: Knex,
   query: KnexQuery,
+  fields: string | string[],
   fromTable: string,
   relationships: RelationshipsJson[] | undefined
 ): KnexQuery {
@@ -114,7 +116,7 @@ function addRelationships(
       query = query.leftJoin(
         toTable,
         `${fromTable}.${from}`,
-        `${relationship.tableName}.${to}`
+        `${toTable}.${to}`
       )
     } else {
       const throughTable = relationship.through
@@ -130,7 +132,7 @@ function addRelationships(
         .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
     }
   }
-  return query
+  return query.limit(BASE_LIMIT)
 }
 
 function buildCreate(
@@ -199,7 +201,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
     [tableName]: query,
   }).select(selectStatement)
   // handle joins
-  return addRelationships(preQuery, tableName, relationships)
+  return addRelationships(knex, preQuery, selectStatement, tableName, relationships)
 }
 
 function buildUpdate(

package/src/integrations/tests/mongo.spec.js

@@ -27,7 +27,7 @@ describe("MongoDB Integration", () => {
     const body = {
       name: "Hello"
     }
-    const response = await config.integration.create({ 
+    await config.integration.create({
       index: indexName,
       json: body,
       extra: { collection: 'testCollection', actionTypes: 'insertOne'}
@@ -54,7 +54,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    const response = await config.integration.delete(query)
+    await config.integration.delete(query)
     expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -65,7 +65,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'updateOne'}
     }
-    const response = await config.integration.update(query)
+    await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -75,10 +75,14 @@ describe("MongoDB Integration", () => {
     const query = {
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    // Weird, need to do an IIFE for jest to recognize that it throws
-    expect(() => config.integration.read(query)()).toThrow(expect.any(Object))
 
+    let error = null
+    try {
+      await config.integration.read(query)
+    } catch (err) {
+      error = err
+    }
+    expect(error).toBeDefined()
     restore()
   })
-
 })

package/src/middleware/currentapp.js

@@ -45,7 +45,7 @@ module.exports = async (ctx, next) => {
     const globalUser = await getCachedSelf(ctx, requestAppId)
     appId = requestAppId
     // retrieving global user gets the right role
-    roleId = globalUser.roleId || BUILTIN_ROLE_IDS.BASIC
+    roleId = globalUser.roleId || roleId
   }
 
   // nothing more to do

package/src/middleware/tests/currentapp.spec.js

@@ -127,8 +127,8 @@ describe("Current app middleware", () => {
       } else {
         expect(cookieFn).not.toHaveBeenCalled()
       }
-      expect(config.ctx.roleId).toEqual("BASIC")
-      expect(config.ctx.user.role._id).toEqual("BASIC")
+      expect(config.ctx.roleId).toEqual("PUBLIC")
+      expect(config.ctx.user.role._id).toEqual("PUBLIC")
       expect(config.ctx.appId).toEqual("app_test")
       expect(config.next).toHaveBeenCalled()
     }
@@ -163,7 +163,7 @@ describe("Current app middleware", () => {
             return "app_test"
           },
           setCookie: jest.fn(),
-          getCookie: () => ({appId: "app_test", roleId: "BASIC"}),
+          getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
         },
         constants: { Cookies: {} },
       }))

package/src/utilities/global.js

@@ -26,7 +26,7 @@ exports.updateAppRole = (appId, user) => {
   if (!user.roleId && user.builder && user.builder.global) {
     user.roleId = BUILTIN_ROLE_IDS.ADMIN
   } else if (!user.roleId) {
-    user.roleId = BUILTIN_ROLE_IDS.BASIC
+    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
   }
   delete user.roles
   return user

package/src/utilities/rowProcessor/index.js

@@ -150,6 +150,10 @@ exports.processAutoColumn = processAutoColumn
  * @returns {object} The coerced value
  */
 exports.coerce = (row, type) => {
+  // no coercion specified for type, skip it
+  if (!TYPE_TRANSFORM_MAP[type]) {
+    return row
+  }
   // eslint-disable-next-line no-prototype-builtins
   if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
     return TYPE_TRANSFORM_MAP[type][row]
@@ -196,6 +200,12 @@ exports.inputProcessing = (
       clonedRow[key] = exports.coerce(value, field.type)
     }
   }
+
+  if (!clonedRow._id || !clonedRow._rev) {
+    clonedRow._id = row._id
+    clonedRow._rev = row._rev
+  }
+
   // handle auto columns - this returns an object like {table, row}
   return processAutoColumn(user, copiedTable, clonedRow, opts)
 }

package/src/utilities/scriptRunner.js

@@ -0,0 +1,21 @@
+const fetch = require("node-fetch")
+const { VM, VMScript } = require("vm2")
+
+class ScriptRunner {
+  constructor(script, context) {
+    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.vm = new VM()
+    this.results = { out: "" }
+    this.vm.setGlobals(context)
+    this.vm.setGlobal("fetch", fetch)
+    this.vm.setGlobal("results", this.results)
+    this.script = new VMScript(code)
+  }
+
+  execute() {
+    this.vm.run(this.script)
+    return this.results.out
+  }
+}
+
+module.exports = ScriptRunner