@budibase/server 0.9.185-alpha.0 → 0.9.185-alpha.12

package/src/{automations/thread.js → threads/automation.js}

@@ -1,5 +1,5 @@
-const actions = require("./actions")
-const automationUtils = require("./automationUtils")
+const actions = require("../automations/actions")
+const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
@@ -8,8 +8,10 @@ const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
 const env = require("../environment")
 const usage = require("../utilities/usageQuota")
+const { definitions: triggerDefs } = require("../automations/triggerInfo")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
@@ -23,6 +25,8 @@ class Orchestrator {
     this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
     this._appId = triggerOutput.appId
     this._app = null
+    const triggerStepId = automation.definition.trigger.stepId
+    triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
     delete triggerOutput.appId
     delete triggerOutput.metadata
@@ -34,11 +38,17 @@ class Orchestrator {
     this._emitter = new AutomationEmitter(this._chainCount + 1)
     this.executionOutput = { trigger: {}, steps: [] }
     // setup the execution output
-    const triggerStepId = automation.definition.trigger.stepId
     const triggerId = automation.definition.trigger.id
     this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput)
   }
 
+  cleanupTriggerOutputs(stepId, triggerOutput) {
+    if (stepId === CRON_STEP_ID) {
+      triggerOutput.timestamp = Date.now()
+    }
+    return triggerOutput
+  }
+
   async getStepFunctionality(stepId) {
     let step = await actions.getAction(stepId)
     if (step == null) {
@@ -119,10 +129,17 @@ class Orchestrator {
   }
 }
 
-module.exports = async job => {
+module.exports = (input, callback) => {
   const automationOrchestrator = new Orchestrator(
-    job.data.automation,
-    job.data.event
+    input.data.automation,
+    input.data.event
   )
-  return automationOrchestrator.execute()
+  automationOrchestrator
+    .execute()
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
 }

package/src/threads/index.js

@@ -0,0 +1,60 @@
+const workerFarm = require("worker-farm")
+const env = require("../environment")
+
+const ThreadType = {
+  QUERY: "query",
+  AUTOMATION: "automation",
+}
+
+function typeToFile(type) {
+  let filename = null
+  switch (type) {
+    case ThreadType.QUERY:
+      filename = "./query"
+      break
+    case ThreadType.AUTOMATION:
+      filename = "./automation"
+      break
+    default:
+      throw "Unknown thread type"
+  }
+  return require.resolve(filename)
+}
+
+class Thread {
+  constructor(type, opts = { timeoutMs: null, count: 1 }) {
+    this.type = type
+    if (!env.isTest()) {
+      const workerOpts = {
+        autoStart: true,
+        maxConcurrentWorkers: opts.count ? opts.count : 1,
+      }
+      if (opts.timeoutMs) {
+        workerOpts.maxCallTime = opts.timeoutMs
+      }
+      this.workers = workerFarm(workerOpts, typeToFile(type))
+    }
+  }
+
+  run(data) {
+    return new Promise((resolve, reject) => {
+      let fncToCall
+      // if in test then don't use threading
+      if (env.isTest()) {
+        fncToCall = require(typeToFile(this.type))
+      } else {
+        fncToCall = this.workers
+      }
+      fncToCall(data, (err, response) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(response)
+        }
+      })
+    })
+  }
+}
+
+module.exports.Thread = Thread
+module.exports.ThreadType = ThreadType

package/src/threads/query.js

@@ -0,0 +1,63 @@
+const ScriptRunner = require("../utilities/scriptRunner")
+const { integrations } = require("../integrations")
+
+function formatResponse(resp) {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+async function runAndTransform(datasource, queryVerb, query, transformer) {
+  const Integration = integrations[datasource.source]
+  if (!Integration) {
+    throw "Integration type does not exist."
+  }
+  const integration = new Integration(datasource.config)
+
+  let rows = formatResponse(await integration[queryVerb](query))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  if (integration.end) {
+    integration.end()
+  }
+
+  return { rows, keys }
+}
+
+module.exports = (input, callback) => {
+  runAndTransform(
+    input.datasource,
+    input.queryVerb,
+    input.query,
+    input.transformer
+  )
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
+}

package/src/utilities/csvParser.js

@@ -51,7 +51,7 @@ function parse(csvString, parsers) {
     })
     result.subscribe(row => {
       // For each CSV row parse all the columns that need parsed
-      for (let key in parsers) {
+      for (let key of Object.keys(parsers)) {
         if (!schema[key] || schema[key].success) {
           // get the validator for the column type
           const validator = VALIDATORS[parsers[key].type]
@@ -76,16 +76,58 @@ function parse(csvString, parsers) {
   })
 }
 
-async function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
+  if (!schema) {
+    return schema
+  }
+  const finalSchema = {}
+  const schemaKeyMap = {}
+  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+  for (let [key, field] of Object.entries(existingTable.schema)) {
+    const lcKey = key.toLowerCase()
+    const foundKey = schemaKeyMap[lcKey]
+    if (foundKey) {
+      finalSchema[key] = schema[foundKey]
+      finalSchema[key].type = field.type
+    }
+  }
+  return finalSchema
+}
+
+async function transform({ schema, csvString, existingTable }) {
   const colParser = {}
 
-  for (let key in schema) {
+  // make sure the table has all the columns required for import
+  if (existingTable) {
+    schema = updateSchema({ schema, existingTable })
+  }
+
+  for (let key of Object.keys(schema)) {
     colParser[key] = PARSERS[schema[key].type] || schema[key].type
   }
 
   try {
-    const json = await csv({ colParser }).fromString(csvString)
-    return json
+    const data = await csv({ colParser }).fromString(csvString)
+    const schemaKeyMap = {}
+    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+    for (let element of data) {
+      if (!data) {
+        continue
+      }
+      for (let key of Object.keys(element)) {
+        const mappedKey = schemaKeyMap[key.toLowerCase()]
+        // isn't a column in the table, remove it
+        if (mappedKey == null) {
+          delete element[key]
+        }
+        // casing is different, fix it in row
+        else if (key !== mappedKey) {
+          element[mappedKey] = element[key]
+          delete element[key]
+        }
+      }
+    }
+    return data
   } catch (err) {
     console.error(`Error transforming CSV to JSON for data import`, err)
     throw err
@@ -95,4 +137,5 @@ async function transform({ schema, csvString }) {
 module.exports = {
   parse,
   transform,
+  updateSchema,
 }

package/src/utilities/index.js

@@ -48,6 +48,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */

package/src/utilities/queue/inMemoryQueue.js

@@ -89,6 +89,13 @@ class InMemoryQueue {
   getRepeatableJobs() {
     return []
   }
+
+  /**
+   * Implemented for tests
+   */
+  async clean() {
+    return []
+  }
 }
 
 module.exports = InMemoryQueue

package/src/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 const fetch = require("node-fetch")
 const { VM, VMScript } = require("vm2")
+const JS_TIMEOUT_MS = 1000
 
 class ScriptRunner {
   constructor(script, context) {
     const code = `let fn = () => {\n${script}\n}; results.out = fn();`
-    this.vm = new VM()
+    this.vm = new VM({
+      timeout: JS_TIMEOUT_MS,
+    })
     this.results = { out: "" }
     this.vm.setGlobals(context)
     this.vm.setGlobal("fetch", fetch)

package/src/utilities/tests/__snapshots__/csvParser.spec.js.snap

@@ -3,19 +3,13 @@
 exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
 Array [
   Object {
-    "Address": "5 Sesame Street",
     "Age": 4324,
-    "Name": "Bertå",
   },
   Object {
-    "Address": "1 World Trade Center",
     "Age": 34,
-    "Name": "Ernie",
   },
   Object {
-    "Address": "44 Second Avenue",
     "Age": 23423,
-    "Name": "Big Bird",
   },
 ]
 `;

package/src/utilities/tests/csvParser.spec.js

@@ -24,6 +24,9 @@ const SCHEMAS = {
     Age: {
       type: "omit",
     },
+    Name: {
+      type: "string",
+    },
   },
   BROKEN: {
     Address: {

package/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "es6",
     "module": "commonjs",
-    "lib": ["es6"],
+    "lib": ["es2019"],
     "allowJs": true,
     "outDir": "dist",
     "strict": true,