@budibase/server 0.9.167-alpha.4 → 0.9.167-alpha.8

package/dist/utilities/scriptRunner.js

@@ -0,0 +1,19 @@
+"use strict";
+const fetch = require("node-fetch");
+const { VM, VMScript } = require("vm2");
+class ScriptRunner {
+    constructor(script, context) {
+        const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
+        this.vm = new VM();
+        this.results = { out: "" };
+        this.vm.setGlobals(context);
+        this.vm.setGlobal("fetch", fetch);
+        this.vm.setGlobal("results", this.results);
+        this.script = new VMScript(code);
+    }
+    execute() {
+        this.vm.run(this.script);
+        return this.results.out;
+    }
+}
+module.exports = ScriptRunner;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.167-alpha.4",
+  "version": "0.9.167-alpha.8",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.167-alpha.4",
-    "@budibase/client": "^0.9.167-alpha.4",
-    "@budibase/string-templates": "^0.9.167-alpha.4",
+    "@budibase/auth": "^0.9.167-alpha.8",
+    "@budibase/client": "^0.9.167-alpha.8",
+    "@budibase/string-templates": "^0.9.167-alpha.8",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -119,6 +119,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -148,5 +149,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "a81ac3293cda24257db1543ba9696ca6d311d51c"
+  "gitHead": "bc396fb840e24afc2bcf01383b493bdd3d783b29"
 }

package/src/api/controllers/query.js

@@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -28,12 +29,39 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  if (!Array.isArray(resp)) {
-    resp = [resp]
-  }
   return resp
 }
 
+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb } = ctx.request.body
-
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-
   const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  // get all the potential fields in the schema
-  const keys = rows.flatMap(Object.keys)
+  const { rows, keys } = await runAndTransform(
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )
 
   ctx.body = {
     rows,
@@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-
   const integration = new Integration(datasource.config)
+
   // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
   // cleanup
   if (integration.end) {
     integration.end()

package/src/api/controllers/script.js

@@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
-
-  ctx.body = executor.execute()
+  const { script, context } = ctx.request.body
+  const runner = new ScriptRunner(script, context)
+  ctx.body = runner.execute()
 }
 
 exports.save = async function (ctx) {

package/src/api/controllers/table/utils.js

@@ -1,5 +1,4 @@
 const CouchDB = require("../../../db")
-const linkRows = require("../../../db/linkedRows")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
@@ -93,19 +92,10 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
         }
       }
 
-      // make sure link rows are up to date
-      finalData.push(
-        linkRows.updateLinks({
-          appId,
-          eventType: linkRows.EventType.ROW_SAVE,
-          row,
-          tableId: row.tableId,
-          table,
-        })
-      )
+      finalData.push(row)
     }
 
-    await db.bulkDocs(await Promise.all(finalData))
+    await db.bulkDocs(finalData)
     let response = await db.put(table)
     table._rev = response._rev
   }

package/src/api/routes/query.js

@@ -31,7 +31,8 @@ function generateQueryValidation() {
     })),
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true)
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
   }))
 }
 
@@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
     parameters: Joi.object({}).required().unknown(true)
   }))
 }

package/src/automations/actions.js

@@ -3,7 +3,6 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const executeScript = require("./steps/executeScript")
-const bash = require("./steps/bash")
 const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const serverLog = require("./steps/serverLog")
@@ -14,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+const env = require("../environment")
 
 const ACTION_IMPLS = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
@@ -22,7 +22,6 @@ const ACTION_IMPLS = {
   DELETE_ROW: deleteRow.run,
   OUTGOING_WEBHOOK: outgoingWebhook.run,
   EXECUTE_SCRIPT: executeScript.run,
-  EXECUTE_BASH: bash.run,
   EXECUTE_QUERY: executeQuery.run,
   SERVER_LOG: serverLog.run,
   DELAY: delay.run,
@@ -42,7 +41,6 @@ const ACTION_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
   EXECUTE_SCRIPT: executeScript.definition,
   EXECUTE_QUERY: executeQuery.definition,
-  EXECUTE_BASH: bash.definition,
   SERVER_LOG: serverLog.definition,
   DELAY: delay.definition,
   FILTER: filter.definition,
@@ -54,6 +52,15 @@ const ACTION_DEFINITIONS = {
   integromat: integromat.definition,
 }
 
+// don't add the bash script/definitions unless in self host
+// the fact this isn't included in any definitions means it cannot be
+// ran at all
+if (env.SELF_HOSTED) {
+  const bash = require("./steps/bash")
+  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
+  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
 /* istanbul ignore next */
 exports.getAction = async function (actionName) {
   if (ACTION_IMPLS[actionName] != null) {

package/src/automations/thread.js

@@ -10,6 +10,7 @@ const env = require("../environment")
 const usage = require("../utilities/usageQuota")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -68,7 +69,13 @@ class Orchestrator {
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
+    let stopped = false
     for (let step of automation.definition.steps) {
+      // execution stopped, record state for that
+      if (stopped) {
+        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+        continue
+      }
       let stepFn = await this.getStepFunctionality(step.stepId)
       step.inputs = await processObject(step.inputs, this._context)
       step.inputs = automationUtils.cleanInputValues(
@@ -86,10 +93,17 @@ class Orchestrator {
             context: this._context,
           })
         })
+        this._context.steps.push(outputs)
+        // if filter causes us to stop execution don't break the loop, set a var
+        // so that we can finish iterating through the steps and record that it stopped
         if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          break
+          stopped = true
+          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+            ...outputs,
+            ...STOPPED_STATUS,
+          })
+          continue
         }
-        this._context.steps.push(outputs)
         this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       } catch (err) {
         console.error(`Automation error - ${step.stepId} - ${err}`)
@@ -99,7 +113,7 @@ class Orchestrator {
 
     // Increment quota for automation runs
     if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
-      usage.update(usage.Properties.AUTOMATION, 1)
+      await usage.update(usage.Properties.AUTOMATION, 1)
     }
     return this.executionOutput
   }

package/src/integrations/tests/mongo.spec.js

@@ -27,7 +27,7 @@ describe("MongoDB Integration", () => {
     const body = {
       name: "Hello"
     }
-    const response = await config.integration.create({ 
+    await config.integration.create({
       index: indexName,
       json: body,
       extra: { collection: 'testCollection', actionTypes: 'insertOne'}
@@ -54,7 +54,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    const response = await config.integration.delete(query)
+    await config.integration.delete(query)
     expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -65,7 +65,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'updateOne'}
     }
-    const response = await config.integration.update(query)
+    await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -75,10 +75,14 @@ describe("MongoDB Integration", () => {
     const query = {
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    // Weird, need to do an IIFE for jest to recognize that it throws
-    expect(() => config.integration.read(query)()).toThrow(expect.any(Object))
 
+    let error = null
+    try {
+      await config.integration.read(query)
+    } catch (err) {
+      error = err
+    }
+    expect(error).toBeDefined()
     restore()
   })
-
 })

package/src/utilities/scriptRunner.js

@@ -0,0 +1,21 @@
+const fetch = require("node-fetch")
+const { VM, VMScript } = require("vm2")
+
+class ScriptRunner {
+  constructor(script, context) {
+    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.vm = new VM()
+    this.results = { out: "" }
+    this.vm.setGlobals(context)
+    this.vm.setGlobal("fetch", fetch)
+    this.vm.setGlobal("results", this.results)
+    this.script = new VMScript(code)
+  }
+
+  execute() {
+    this.vm.run(this.script)
+    return this.results.out
+  }
+}
+
+module.exports = ScriptRunner