@budibase/server 0.9.188 → 0.9.190-alpha.2

package/src/integrations/tests/oracle.spec.js

@@ -0,0 +1,94 @@
+const oracledb = require("oracledb")
+const OracleIntegration = require("../oracle")
+jest.mock("oracledb")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new OracleIntegration.integration(config) 
+  }
+}
+
+const options = { autoCommit: true }
+
+describe("Oracle Integration", () => {
+  let config 
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    expect(oracledb.closeMock).toHaveBeenCalled()
+    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the create method with the correct params", async () => {
+    const sql = "insert into users (name, age) values ('Joe', 123);"
+    await config.integration.create({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const sql = "select * from users;"
+    await config.integration.read({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the update method with the correct params", async () => {
+    const sql = "update table users set name = 'test';"
+    const response = await config.integration.update({ 
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const sql = "delete from users where name = 'todelete';"
+    await config.integration.delete({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  describe("no rows returned", () => {
+    beforeEach(() => {
+      oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
+    })
+
+    it("returns the correct response when the create response has no rows", async () => {
+    const sql = "insert into users (name, age) values ('Joe', 123);"
+      const response = await config.integration.create({ 
+        sql
+      })
+      expect(response).toEqual([{ created: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+
+    it("returns the correct response when the update response has no rows", async () => {
+      const sql = "update table users set name = 'test';"
+      const response = await config.integration.update({ 
+        sql
+      })
+      expect(response).toEqual([{ updated: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+
+    it("returns the correct response when the delete response has no rows", async () => {
+      const sql = "delete from users where name = 'todelete';"
+      const response = await config.integration.delete({ 
+        sql
+      })
+      expect(response).toEqual([{ deleted: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+  })
+})

package/src/integrations/tests/postgres.spec.js

@@ -15,6 +15,10 @@ describe("Postgres Integration", () => {
     config = new TestConfiguration()
   })
 
+  it("calls the connection callback", async () => {
+    expect(pg.on).toHaveBeenCalledWith('connect', expect.anything())
+  })
+
   it("calls the create method with the correct params", async () => {
     const sql = "insert into users (name, age) values ('Joe', 123);"
     await config.integration.create({

package/src/integrations/utils.ts

@@ -1,12 +1,54 @@
 import { SqlQuery } from "../definitions/datasource"
-import { Datasource } from "../definitions/common"
+import { Datasource, Table } from "../definitions/common"
 import { SourceNames } from "../definitions/datasource"
 const { DocumentTypes, SEPARATOR } = require("../db/utils")
-const { FieldTypes } = require("../constants")
+const {
+  FieldTypes,
+  BuildSchemaErrors,
+  InvalidColumns,
+} = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 
+const SQL_TYPE_MAP = {
+  text: FieldTypes.LONGFORM,
+  varchar: FieldTypes.STRING,
+  integer: FieldTypes.NUMBER,
+  bigint: FieldTypes.NUMBER,
+  decimal: FieldTypes.NUMBER,
+  smallint: FieldTypes.NUMBER,
+  real: FieldTypes.NUMBER,
+  "double precision": FieldTypes.NUMBER,
+  timestamp: FieldTypes.DATETIME,
+  time: FieldTypes.DATETIME,
+  boolean: FieldTypes.BOOLEAN,
+  json: FieldTypes.JSON,
+  date: FieldTypes.DATETIME,
+  blob: FieldTypes.LONGFORM,
+  enum: FieldTypes.STRING,
+  float: FieldTypes.NUMBER,
+  int: FieldTypes.NUMBER,
+  numeric: FieldTypes.NUMBER,
+  mediumint: FieldTypes.NUMBER,
+  dec: FieldTypes.NUMBER,
+  double: FieldTypes.NUMBER,
+  fixed: FieldTypes.NUMBER,
+  datetime: FieldTypes.DATETIME,
+  tinyint: FieldTypes.BOOLEAN,
+  long: FieldTypes.LONGFORM,
+  number: FieldTypes.NUMBER,
+  binary_float: FieldTypes.NUMBER,
+  binary_double: FieldTypes.NUMBER,
+}
+
+export enum SqlClients {
+  MS_SQL = "mssql",
+  POSTGRES = "pg",
+  MY_SQL = "mysql",
+  ORACLE = "oracledb",
+}
+
 export function isExternalTable(tableId: string) {
   return tableId.includes(DocumentTypes.DATASOURCE)
 }
@@ -68,8 +110,8 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function convertType(type: string, map: { [key: string]: any }) {
-  for (let [external, internal] of Object.entries(map)) {
+export function convertSqlType(type: string) {
+  for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
     if (type.toLowerCase().includes(external)) {
       return internal
     }
@@ -102,14 +144,14 @@ export function isIsoDateString(str: string) {
 }
 
 // add the existing relationships from the entities if they exist, to prevent them from being overridden
-export function copyExistingPropsOver(
+function copyExistingPropsOver(
   tableName: string,
-  tables: { [key: string]: any },
+  table: Table,
   entities: { [key: string]: any }
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
-      tables[tableName].primaryDisplay = entities[tableName].primaryDisplay
+      table.primaryDisplay = entities[tableName].primaryDisplay
     }
     const existingTableSchema = entities[tableName].schema
     for (let key in existingTableSchema) {
@@ -117,8 +159,36 @@ export function copyExistingPropsOver(
         continue
       }
       if (existingTableSchema[key].type === "link") {
-        tables[tableName].schema[key] = existingTableSchema[key]
+        table.schema[key] = existingTableSchema[key]
       }
     }
   }
+  return table
+}
+
+export function finaliseExternalTables(
+  tables: { [key: string]: any },
+  entities: { [key: string]: any }
+) {
+  const invalidColumns = Object.values(InvalidColumns)
+  let finalTables: { [key: string]: any } = {}
+  const errors: { [key: string]: string } = {}
+  for (let [name, table] of Object.entries(tables)) {
+    const schemaFields = Object.keys(table.schema)
+    // make sure every table has a key
+    if (table.primary == null || table.primary.length === 0) {
+      errors[name] = BuildSchemaErrors.NO_KEY
+      continue
+    } else if (schemaFields.find(field => invalidColumns.includes(field))) {
+      errors[name] = BuildSchemaErrors.INVALID_COLUMN
+      continue
+    }
+    // make sure all previous props have been added back
+    finalTables[name] = copyExistingPropsOver(name, table, entities)
+  }
+  // sort the tables by name
+  finalTables = Object.entries(finalTables)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .reduce((r, [k, v]) => ({ ...r, [k]: v }), {})
+  return { tables: finalTables, errors }
 }

package/src/middleware/authorized.js

@@ -18,7 +18,8 @@ module.exports =
   (permType, permLevel = null) =>
   async (ctx, next) => {
     // webhooks don't need authentication, each webhook unique
-    if (isWebhookEndpoint(ctx)) {
+    // also internal requests (between services) don't need authorized
+    if (isWebhookEndpoint(ctx) || ctx.internal) {
       return next()
     }
 

package/src/middleware/builder.js

@@ -8,6 +8,7 @@ const {
 const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/auth/permissions")
+const { app: appCache } = require("@budibase/auth/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -50,7 +51,9 @@ async function updateAppUpdatedAt(ctx) {
   const db = new CouchDB(appId)
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
-  await db.put(metadata)
+  const response = await db.put(metadata)
+  metadata._rev = response.rev
+  await appCache.invalidateAppMetadata(appId, metadata)
   // set a new debounce record with a short TTL
   await setDebounce(appId, DEBOUNCE_TIME_SEC)
 }

package/src/tests/utilities/TestConfiguration.js

@@ -311,7 +311,8 @@ class TestConfiguration {
 
   async createDatasource(config = null) {
     config = config || basicDatasource()
-    this.datasource = await this._req(config, null, controllers.datasource.save)
+    const response = await this._req(config, null, controllers.datasource.save)
+    this.datasource = response.datasource
     return this.datasource
   }
 

package/src/{automations/thread.js → threads/automation.js}

@@ -1,15 +1,19 @@
-const actions = require("./actions")
-const automationUtils = require("./automationUtils")
+// when thread starts, make sure it is recorded
+const env = require("../environment")
+env.setInThread()
+const actions = require("../automations/actions")
+const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
 const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
-const env = require("../environment")
 const usage = require("../utilities/usageQuota")
+const { definitions: triggerDefs } = require("../automations/triggerInfo")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
@@ -23,6 +27,8 @@ class Orchestrator {
     this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
     this._appId = triggerOutput.appId
     this._app = null
+    const triggerStepId = automation.definition.trigger.stepId
+    triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
     delete triggerOutput.appId
     delete triggerOutput.metadata
@@ -34,11 +40,17 @@ class Orchestrator {
     this._emitter = new AutomationEmitter(this._chainCount + 1)
     this.executionOutput = { trigger: {}, steps: [] }
     // setup the execution output
-    const triggerStepId = automation.definition.trigger.stepId
     const triggerId = automation.definition.trigger.id
     this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput)
   }
 
+  cleanupTriggerOutputs(stepId, triggerOutput) {
+    if (stepId === CRON_STEP_ID) {
+      triggerOutput.timestamp = Date.now()
+    }
+    return triggerOutput
+  }
+
   async getStepFunctionality(stepId) {
     let step = await actions.getAction(stepId)
     if (step == null) {
@@ -119,10 +131,17 @@ class Orchestrator {
   }
 }
 
-module.exports = async job => {
+module.exports = (input, callback) => {
   const automationOrchestrator = new Orchestrator(
-    job.data.automation,
-    job.data.event
+    input.data.automation,
+    input.data.event
   )
-  return automationOrchestrator.execute()
+  automationOrchestrator
+    .execute()
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
 }

package/src/threads/index.js

@@ -0,0 +1,66 @@
+const workerFarm = require("worker-farm")
+const env = require("../environment")
+
+const ThreadType = {
+  QUERY: "query",
+  AUTOMATION: "automation",
+}
+
+function typeToFile(type) {
+  let filename = null
+  switch (type) {
+    case ThreadType.QUERY:
+      filename = "./query"
+      break
+    case ThreadType.AUTOMATION:
+      filename = "./automation"
+      break
+    default:
+      throw "Unknown thread type"
+  }
+  return require.resolve(filename)
+}
+
+class Thread {
+  constructor(type, opts = { timeoutMs: null, count: 1 }) {
+    this.type = type
+    this.count = opts.count ? opts.count : 1
+    this.disableThreading =
+      env.isTest() ||
+      env.DISABLE_THREADING ||
+      this.count === 0 ||
+      env.isInThread()
+    if (!this.disableThreading) {
+      const workerOpts = {
+        autoStart: true,
+        maxConcurrentWorkers: this.count,
+      }
+      if (opts.timeoutMs) {
+        workerOpts.maxCallTime = opts.timeoutMs
+      }
+      this.workers = workerFarm(workerOpts, typeToFile(type))
+    }
+  }
+
+  run(data) {
+    return new Promise((resolve, reject) => {
+      let fncToCall
+      // if in test then don't use threading
+      if (this.disableThreading) {
+        fncToCall = require(typeToFile(this.type))
+      } else {
+        fncToCall = this.workers
+      }
+      fncToCall(data, (err, response) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(response)
+        }
+      })
+    })
+  }
+}
+
+module.exports.Thread = Thread
+module.exports.ThreadType = ThreadType

package/src/threads/query.js

@@ -0,0 +1,66 @@
+// when thread starts, make sure it is recorded
+const env = require("../environment")
+env.setInThread()
+const ScriptRunner = require("../utilities/scriptRunner")
+const { integrations } = require("../integrations")
+
+function formatResponse(resp) {
+  if (typeof resp === "string") {
+    try {
+      resp = JSON.parse(resp)
+    } catch (err) {
+      resp = { response: resp }
+    }
+  }
+  return resp
+}
+
+async function runAndTransform(datasource, queryVerb, query, transformer) {
+  const Integration = integrations[datasource.source]
+  if (!Integration) {
+    throw "Integration type does not exist."
+  }
+  const integration = new Integration(datasource.config)
+
+  let rows = formatResponse(await integration[queryVerb](query))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  if (integration.end) {
+    integration.end()
+  }
+
+  return { rows, keys }
+}
+
+module.exports = (input, callback) => {
+  runAndTransform(
+    input.datasource,
+    input.queryVerb,
+    input.query,
+    input.transformer
+  )
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
+}

package/src/utilities/csvParser.js

@@ -51,7 +51,7 @@ function parse(csvString, parsers) {
     })
     result.subscribe(row => {
       // For each CSV row parse all the columns that need parsed
-      for (let key in parsers) {
+      for (let key of Object.keys(parsers)) {
         if (!schema[key] || schema[key].success) {
           // get the validator for the column type
           const validator = VALIDATORS[parsers[key].type]
@@ -76,16 +76,58 @@ function parse(csvString, parsers) {
   })
 }
 
-async function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
+  if (!schema) {
+    return schema
+  }
+  const finalSchema = {}
+  const schemaKeyMap = {}
+  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+  for (let [key, field] of Object.entries(existingTable.schema)) {
+    const lcKey = key.toLowerCase()
+    const foundKey = schemaKeyMap[lcKey]
+    if (foundKey) {
+      finalSchema[key] = schema[foundKey]
+      finalSchema[key].type = field.type
+    }
+  }
+  return finalSchema
+}
+
+async function transform({ schema, csvString, existingTable }) {
   const colParser = {}
 
-  for (let key in schema) {
+  // make sure the table has all the columns required for import
+  if (existingTable) {
+    schema = updateSchema({ schema, existingTable })
+  }
+
+  for (let key of Object.keys(schema)) {
     colParser[key] = PARSERS[schema[key].type] || schema[key].type
   }
 
   try {
-    const json = await csv({ colParser }).fromString(csvString)
-    return json
+    const data = await csv({ colParser }).fromString(csvString)
+    const schemaKeyMap = {}
+    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+    for (let element of data) {
+      if (!data) {
+        continue
+      }
+      for (let key of Object.keys(element)) {
+        const mappedKey = schemaKeyMap[key.toLowerCase()]
+        // isn't a column in the table, remove it
+        if (mappedKey == null) {
+          delete element[key]
+        }
+        // casing is different, fix it in row
+        else if (key !== mappedKey) {
+          element[mappedKey] = element[key]
+          delete element[key]
+        }
+      }
+    }
+    return data
   } catch (err) {
     console.error(`Error transforming CSV to JSON for data import`, err)
     throw err
@@ -95,4 +137,5 @@ async function transform({ schema, csvString }) {
 module.exports = {
   parse,
   transform,
+  updateSchema,
 }

package/src/utilities/global.js

@@ -46,9 +46,13 @@ exports.getCachedSelf = async (ctx, appId) => {
   return processUser(appId, user)
 }
 
-exports.getGlobalUser = async (appId, userId) => {
+exports.getRawGlobalUser = async userId => {
   const db = getGlobalDB()
-  let user = await db.get(getGlobalIDFromUserMetadataID(userId))
+  return db.get(getGlobalIDFromUserMetadataID(userId))
+}
+
+exports.getGlobalUser = async (appId, userId) => {
+  let user = await exports.getRawGlobalUser(userId)
   return processUser(appId, user)
 }
 
@@ -73,6 +77,7 @@ exports.getGlobalUsers = async (appId = null, users = null) => {
     .filter(user => user != null)
     .map(user => {
       delete user.password
+      delete user.forceResetPassword
       return user
     })
   if (!appId) {

package/src/utilities/index.js

@@ -48,6 +48,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
@@ -139,3 +140,13 @@ exports.stringToReadStream = string => {
     },
   })
 }
+
+exports.doesDatabaseExist = async dbName => {
+  try {
+    const db = new CouchDB(dbName, { skip_setup: true })
+    const info = await db.info()
+    return info && !info.error
+  } catch (err) {
+    return false
+  }
+}

package/src/utilities/queue/inMemoryQueue.js

@@ -89,6 +89,13 @@ class InMemoryQueue {
   getRepeatableJobs() {
     return []
   }
+
+  /**
+   * Implemented for tests
+   */
+  async clean() {
+    return []
+  }
 }
 
 module.exports = InMemoryQueue

package/src/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 const fetch = require("node-fetch")
 const { VM, VMScript } = require("vm2")
+const JS_TIMEOUT_MS = 1000
 
 class ScriptRunner {
   constructor(script, context) {
     const code = `let fn = () => {\n${script}\n}; results.out = fn();`
-    this.vm = new VM()
+    this.vm = new VM({
+      timeout: JS_TIMEOUT_MS,
+    })
     this.results = { out: "" }
     this.vm.setGlobals(context)
     this.vm.setGlobal("fetch", fetch)

package/src/utilities/tests/__snapshots__/csvParser.spec.js.snap

@@ -3,19 +3,13 @@
 exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
 Array [
   Object {
-    "Address": "5 Sesame Street",
     "Age": 4324,
-    "Name": "Bertå",
   },
   Object {
-    "Address": "1 World Trade Center",
     "Age": 34,
-    "Name": "Ernie",
   },
   Object {
-    "Address": "44 Second Avenue",
     "Age": 23423,
-    "Name": "Big Bird",
   },
 ]
 `;

package/src/utilities/tests/csvParser.spec.js

@@ -24,6 +24,9 @@ const SCHEMAS = {
     Age: {
       type: "omit",
     },
+    Name: {
+      type: "string",
+    },
   },
   BROKEN: {
     Address: {

package/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "es6",
     "module": "commonjs",
-    "lib": ["es6"],
+    "lib": ["es2019"],
     "allowJs": true,
     "outDir": "dist",
     "strict": true,