@budibase/server 0.9.185-alpha.2 → 0.9.185-alpha.20

package/src/{automations/thread.js → threads/automation.js}

@@ -1,15 +1,19 @@
-const actions = require("./actions")
-const automationUtils = require("./automationUtils")
+// when thread starts, make sure it is recorded
+const env = require("../environment")
+env.setInThread()
+const actions = require("../automations/actions")
+const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
 const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
-const env = require("../environment")
 const usage = require("../utilities/usageQuota")
+const { definitions: triggerDefs } = require("../automations/triggerInfo")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
@@ -23,6 +27,8 @@ class Orchestrator {
     this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
     this._appId = triggerOutput.appId
     this._app = null
+    const triggerStepId = automation.definition.trigger.stepId
+    triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
     delete triggerOutput.appId
     delete triggerOutput.metadata
@@ -34,11 +40,17 @@ class Orchestrator {
     this._emitter = new AutomationEmitter(this._chainCount + 1)
     this.executionOutput = { trigger: {}, steps: [] }
     // setup the execution output
-    const triggerStepId = automation.definition.trigger.stepId
     const triggerId = automation.definition.trigger.id
     this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput)
   }
 
+  cleanupTriggerOutputs(stepId, triggerOutput) {
+    if (stepId === CRON_STEP_ID) {
+      triggerOutput.timestamp = Date.now()
+    }
+    return triggerOutput
+  }
+
   async getStepFunctionality(stepId) {
     let step = await actions.getAction(stepId)
     if (step == null) {
@@ -119,10 +131,17 @@ class Orchestrator {
   }
 }
 
-module.exports = async job => {
+module.exports = (input, callback) => {
   const automationOrchestrator = new Orchestrator(
-    job.data.automation,
-    job.data.event
+    input.data.automation,
+    input.data.event
   )
-  return automationOrchestrator.execute()
+  automationOrchestrator
+    .execute()
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
 }

package/src/threads/index.js

@@ -0,0 +1,66 @@
+const workerFarm = require("worker-farm")
+const env = require("../environment")
+
+const ThreadType = {
+  QUERY: "query",
+  AUTOMATION: "automation",
+}
+
+function typeToFile(type) {
+  let filename = null
+  switch (type) {
+    case ThreadType.QUERY:
+      filename = "./query"
+      break
+    case ThreadType.AUTOMATION:
+      filename = "./automation"
+      break
+    default:
+      throw "Unknown thread type"
+  }
+  return require.resolve(filename)
+}
+
+class Thread {
+  constructor(type, opts = { timeoutMs: null, count: 1 }) {
+    this.type = type
+    this.count = opts.count ? opts.count : 1
+    this.disableThreading =
+      env.isTest() ||
+      env.DISABLE_THREADING ||
+      this.count === 0 ||
+      env.isInThread()
+    if (!this.disableThreading) {
+      const workerOpts = {
+        autoStart: true,
+        maxConcurrentWorkers: this.count,
+      }
+      if (opts.timeoutMs) {
+        workerOpts.maxCallTime = opts.timeoutMs
+      }
+      this.workers = workerFarm(workerOpts, typeToFile(type))
+    }
+  }
+
+  run(data) {
+    return new Promise((resolve, reject) => {
+      let fncToCall
+      // if in test then don't use threading
+      if (this.disableThreading) {
+        fncToCall = require(typeToFile(this.type))
+      } else {
+        fncToCall = this.workers
+      }
+      fncToCall(data, (err, response) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(response)
+        }
+      })
+    })
+  }
+}
+
+module.exports.Thread = Thread
+module.exports.ThreadType = ThreadType

package/src/threads/query.js

@@ -0,0 +1,66 @@
+// when thread starts, make sure it is recorded
+const env = require("../environment")
+env.setInThread()
+const ScriptRunner = require("../utilities/scriptRunner")
+const { integrations } = require("../integrations")
+
+function formatResponse(resp) {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+async function runAndTransform(datasource, queryVerb, query, transformer) {
+  const Integration = integrations[datasource.source]
+  if (!Integration) {
+    throw "Integration type does not exist."
+  }
+  const integration = new Integration(datasource.config)
+
+  let rows = formatResponse(await integration[queryVerb](query))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  if (integration.end) {
+    integration.end()
+  }
+
+  return { rows, keys }
+}
+
+module.exports = (input, callback) => {
+  runAndTransform(
+    input.datasource,
+    input.queryVerb,
+    input.query,
+    input.transformer
+  )
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
+}

package/src/utilities/index.js

@@ -48,6 +48,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */

package/src/utilities/queue/inMemoryQueue.js

@@ -89,6 +89,13 @@ class InMemoryQueue {
   getRepeatableJobs() {
     return []
   }
+
+  /**
+   * Implemented for tests
+   */
+  async clean() {
+    return []
+  }
 }
 
 module.exports = InMemoryQueue

package/src/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 const fetch = require("node-fetch")
 const { VM, VMScript } = require("vm2")
+const JS_TIMEOUT_MS = 1000
 
 class ScriptRunner {
   constructor(script, context) {
     const code = `let fn = () => {\n${script}\n}; results.out = fn();`
-    this.vm = new VM()
+    this.vm = new VM({
+      timeout: JS_TIMEOUT_MS,
+    })
     this.results = { out: "" }
     this.vm.setGlobals(context)
     this.vm.setGlobal("fetch", fetch)

package/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "es6",
     "module": "commonjs",
-    "lib": ["es6"],
+    "lib": ["es2019"],
     "allowJs": true,
     "outDir": "dist",
     "strict": true,