@budibase/server 2.1.40-alpha.5 → 2.1.41

package/src/utilities/queue/inMemoryQueue.js

@@ -0,0 +1,122 @@
+let events = require("events")
+
+/**
+ * Bull works with a Job wrapper around all messages that contains a lot more information about
+ * the state of the message, this object constructor implements the same schema of Bull jobs
+ * for the sake of maintaining API consistency.
+ * @param {string} queue The name of the queue which the message will be carried on.
+ * @param {object} message The JSON message which will be passed back to the consumer.
+ * @returns {Object} A new job which can now be put onto the queue, this is mostly an
+ * internal structure so that an in memory queue can be easily swapped for a Bull queue.
+ */
+function newJob(queue, message) {
+  return {
+    timestamp: Date.now(),
+    queue: queue,
+    data: message,
+  }
+}
+
+/**
+ * This is designed to replicate Bull (https://github.com/OptimalBits/bull) in memory as a sort of mock.
+ * It is relatively simple, using an event emitter internally to register when messages are available
+ * to the consumers - in can support many inputs and many consumers.
+ */
+class InMemoryQueue {
+  /**
+   * The constructor the queue, exactly the same as that of Bulls.
+   * @param {string} name The name of the queue which is being configured.
+   * @param {object|null} opts This is not used by the in memory queue as there is no real use
+   * case when in memory, but is the same API as Bull
+   */
+  constructor(name, opts = null) {
+    this._name = name
+    this._opts = opts
+    this._messages = []
+    this._emitter = new events.EventEmitter()
+  }
+
+  /**
+   * Same callback API as Bull, each callback passed to this will consume messages as they are
+   * available. Please note this is a queue service, not a notification service, so each
+   * consumer will receive different messages.
+   * @param {function<object>} func The callback function which will return a "Job", the same
+   * as the Bull API, within this job the property "data" contains the JSON message. Please
+   * note this is incredibly limited compared to Bull as in reality the Job would contain
+   * a lot more information about the queue and current status of Bull cluster.
+   */
+  process(func) {
+    this._emitter.on("message", async () => {
+      if (this._messages.length <= 0) {
+        return
+      }
+      let msg = this._messages.shift()
+      let resp = func(msg)
+      if (resp.then != null) {
+        await resp
+      }
+    })
+  }
+
+  // simply puts a message to the queue and emits to the queue for processing
+  /**
+   * Simple function to replicate the add message functionality of Bull, putting
+   * a new message on the queue. This then emits an event which will be used to
+   * return the message to a consumer (if one is attached).
+   * @param {object} msg A message to be transported over the queue, this should be
+   * a JSON message as this is required by Bull.
+   */
+  // eslint-disable-next-line no-unused-vars
+  add(msg, repeat) {
+    if (typeof msg !== "object") {
+      throw "Queue only supports carrying JSON."
+    }
+    this._messages.push(newJob(this._name, msg))
+    this._emitter.emit("message")
+  }
+
+  /**
+   * replicating the close function from bull, which waits for jobs to finish.
+   */
+  async close() {
+    return []
+  }
+
+  /**
+   * This removes a cron which has been implemented, this is part of Bull API.
+   * @param {string} cronJobId The cron which is to be removed.
+   */
+  removeRepeatableByKey(cronJobId) {
+    // TODO: implement for testing
+    console.log(cronJobId)
+  }
+
+  /**
+   * Implemented for tests
+   */
+  getRepeatableJobs() {
+    return []
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  removeJobs(pattern) {
+    // no-op
+  }
+
+  /**
+   * Implemented for tests
+   */
+  async clean() {
+    return []
+  }
+
+  async getJob() {
+    return {}
+  }
+
+  on() {
+    // do nothing
+  }
+}
+
+module.exports = InMemoryQueue

package/src/utilities/{redis.ts → redis.js}

@@ -1,47 +1,46 @@
-import { redis } from "@budibase/backend-core"
-import { getGlobalIDFromUserMetadataID } from "../db/utils"
-import { ContextUser } from "@budibase/types"
+const { Client, utils } = require("@budibase/backend-core/redis")
+const { getGlobalIDFromUserMetadataID } = require("../db/utils")
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient, debounceClient, flagClient
 
 // we init this as we want to keep the connection open all the time
 // reduces the performance hit
-export async function init() {
-  devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
-  debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
-  flagClient = new redis.Client(redis.utils.Databases.FLAGS)
+exports.init = async () => {
+  devAppClient = new Client(utils.Databases.DEV_LOCKS)
+  debounceClient = new Client(utils.Databases.DEBOUNCE)
+  flagClient = new Client(utils.Databases.FLAGS)
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
 }
 
-export async function shutdown() {
+exports.shutdown = async () => {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
   console.log("Redis shutdown")
 }
 
-export async function doesUserHaveLock(devAppId: string, user: ContextUser) {
+exports.doesUserHaveLock = async (devAppId, user) => {
   const value = await devAppClient.get(devAppId)
   if (!value) {
     return true
   }
   // make sure both IDs are global
   const expected = getGlobalIDFromUserMetadataID(value._id)
-  const userId = getGlobalIDFromUserMetadataID(user._id!)
+  const userId = getGlobalIDFromUserMetadataID(user._id)
   return expected === userId
 }
 
-export async function getLocksById(appIds: string[]) {
+exports.getLocksById = async appIds => {
   return await devAppClient.bulkGet(appIds)
 }
 
-export async function updateLock(devAppId: string, user: ContextUser) {
+exports.updateLock = async (devAppId, user) => {
   // make sure always global user ID
-  const globalId = getGlobalIDFromUserMetadataID(user._id!)
+  const globalId = getGlobalIDFromUserMetadataID(user._id)
   const inputUser = {
     ...user,
     userId: globalId,
@@ -52,35 +51,35 @@ export async function updateLock(devAppId: string, user: ContextUser) {
   await devAppClient.store(devAppId, inputUser, APP_DEV_LOCK_SECONDS)
 }
 
-export async function clearLock(devAppId: string, user: ContextUser) {
+exports.clearLock = async (devAppId, user) => {
   const value = await devAppClient.get(devAppId)
   if (!value) {
     return
   }
-  const userId = getGlobalIDFromUserMetadataID(user._id!)
+  const userId = getGlobalIDFromUserMetadataID(user._id)
   if (value._id !== userId) {
     throw "User does not hold lock, cannot clear it."
   }
   await devAppClient.delete(devAppId)
 }
 
-export async function checkDebounce(id: string) {
+exports.checkDebounce = async id => {
   return debounceClient.get(id)
 }
 
-export async function setDebounce(id: string, seconds: number) {
+exports.setDebounce = async (id, seconds) => {
   await debounceClient.store(id, "debouncing", seconds)
 }
 
-export async function setTestFlag(id: string) {
+exports.setTestFlag = async id => {
   await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS)
 }
 
-export async function checkTestFlag(id: string) {
+exports.checkTestFlag = async id => {
   const flag = await flagClient.get(id)
   return !!(flag && flag.testing)
 }
 
-export async function clearTestFlag(id: string) {
+exports.clearTestFlag = async id => {
   await devAppClient.delete(id)
 }

package/src/utilities/routing/index.js

@@ -0,0 +1,25 @@
+const { createRoutingView } = require("../../db/views/staticViews")
+const { ViewName, getQueryIndex, UNICODE_MAX } = require("../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+exports.getRoutingInfo = async () => {
+  const db = getAppDB()
+  try {
+    const allRouting = await db.query(getQueryIndex(ViewName.ROUTING), {
+      startKey: "",
+      endKey: UNICODE_MAX,
+    })
+    return allRouting.rows.map(row => row.value)
+  } catch (err) {
+    // check if the view doesn't exist, it should for all new instances
+    /* istanbul ignore next */
+    if (err != null && err.name === "not_found") {
+      await createRoutingView()
+      return exports.getRoutingInfo()
+    } else {
+      throw err
+    }
+  }
+}
+
+exports.createRoutingView = createRoutingView

package/src/utilities/rowProcessor/{index.ts → index.js}

@@ -1,41 +1,129 @@
-import * as linkRows from "../../db/linkedRows"
-import { FieldTypes, AutoFieldSubTypes } from "../../constants"
-import { attachmentsRelativeURL } from "../index"
-import { processFormulas, fixAutoColumnSubType } from "./utils"
-import { ObjectStoreBuckets } from "../../constants"
-import { context, db as dbCore, objectStore } from "@budibase/backend-core"
-import { InternalTables } from "../../db/utils"
-import { TYPE_TRANSFORM_MAP } from "./map"
-import { Row, Table, ContextUser } from "@budibase/types"
+const linkRows = require("../../db/linkedRows")
 const { cloneDeep } = require("lodash/fp")
-export * from "./utils"
-
-type AutoColumnProcessingOpts = {
-  reprocessing?: boolean
-  noAutoRelationships?: boolean
-}
+const { FieldTypes, AutoFieldSubTypes } = require("../../constants")
+const { attachmentsRelativeURL } = require("../index")
+const { processFormulas, fixAutoColumnSubType } = require("./utils")
+const { deleteFiles } = require("../../utilities/fileSystem/utilities")
+const { ObjectStoreBuckets } = require("../../constants")
+const {
+  isProdAppID,
+  getProdAppID,
+  dbExists,
+} = require("@budibase/backend-core/db")
+const { getAppId } = require("@budibase/backend-core/context")
+const { InternalTables } = require("../../db/utils")
 
 const BASE_AUTO_ID = 1
 
+/**
+ * A map of how we convert various properties in rows to each other based on the row type.
+ */
+const TYPE_TRANSFORM_MAP = {
+  [FieldTypes.LINK]: {
+    "": [],
+    [null]: [],
+    [undefined]: undefined,
+    parse: link => {
+      if (Array.isArray(link) && typeof link[0] === "object") {
+        return link.map(el => (el && el._id ? el._id : el))
+      }
+      if (typeof link === "string") {
+        return [link]
+      }
+      return link
+    },
+  },
+  [FieldTypes.OPTIONS]: {
+    "": null,
+    [null]: null,
+    [undefined]: undefined,
+  },
+  [FieldTypes.ARRAY]: {
+    "": [],
+    [null]: [],
+    [undefined]: undefined,
+  },
+  [FieldTypes.STRING]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.BARCODEQR]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.FORMULA]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.LONGFORM]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
+  [FieldTypes.NUMBER]: {
+    "": null,
+    [null]: null,
+    [undefined]: undefined,
+    parse: n => parseFloat(n),
+  },
+  [FieldTypes.DATETIME]: {
+    "": null,
+    [undefined]: undefined,
+    [null]: null,
+    parse: date => {
+      if (date instanceof Date) {
+        return date.toISOString()
+      }
+      return date
+    },
+  },
+  [FieldTypes.ATTACHMENT]: {
+    "": [],
+    [null]: [],
+    [undefined]: undefined,
+  },
+  [FieldTypes.BOOLEAN]: {
+    "": null,
+    [null]: null,
+    [undefined]: undefined,
+    true: true,
+    false: false,
+  },
+  [FieldTypes.AUTO]: {
+    parse: () => undefined,
+  },
+  [FieldTypes.JSON]: {
+    parse: input => {
+      try {
+        if (input === "") {
+          return undefined
+        }
+        return JSON.parse(input)
+      } catch (err) {
+        return input
+      }
+    },
+  },
+}
+
 /**
  * Given the old state of the row and the new one after an update, this will
  * find the keys that have been removed in the updated row.
  */
-function getRemovedAttachmentKeys(
-  oldRow: Row,
-  row: Row,
-  attachmentKey: string
-) {
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
   if (!oldRow[attachmentKey]) {
     return []
   }
-  const oldKeys = oldRow[attachmentKey].map((attachment: any) => attachment.key)
+  const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key)
   // no attachments in new row, all removed
   if (!row[attachmentKey]) {
     return oldKeys
   }
-  const newKeys = row[attachmentKey].map((attachment: any) => attachment.key)
-  return oldKeys.filter((key: any) => newKeys.indexOf(key) === -1)
+  const newKeys = row[attachmentKey].map(attachment => attachment.key)
+  return oldKeys.filter(key => newKeys.indexOf(key) === -1)
 }
 
 /**
@@ -48,11 +136,11 @@ function getRemovedAttachmentKeys(
  * @returns {{row: Object, table: Object}} The updated row and table, the table may need to be updated
  * for automatic ID purposes.
  */
-export function processAutoColumn(
-  user: ContextUser | null,
-  table: Table,
-  row: Row,
-  opts?: AutoColumnProcessingOpts
+function processAutoColumn(
+  user,
+  table,
+  row,
+  opts = { reprocessing: false, noAutoRelationships: false }
 ) {
   let noUser = !user || !user.userId
   let isUserTable = table._id === InternalTables.USER_METADATA
@@ -61,7 +149,7 @@ export function processAutoColumn(
   const creating = !row._rev
   // check its not user table, or whether any of the processing options have been disabled
   const shouldUpdateUserFields =
-    !isUserTable && !opts?.reprocessing && !opts?.noAutoRelationships && !noUser
+    !isUserTable && !opts.reprocessing && !opts.noAutoRelationships && !noUser
   for (let [key, schema] of Object.entries(table.schema)) {
     if (!schema.autocolumn) {
       continue
@@ -71,7 +159,7 @@ export function processAutoColumn(
     }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
-        if (creating && shouldUpdateUserFields && user) {
+        if (creating && shouldUpdateUserFields) {
           row[key] = [user.userId]
         }
         break
@@ -81,7 +169,7 @@ export function processAutoColumn(
         }
         break
       case AutoFieldSubTypes.UPDATED_BY:
-        if (shouldUpdateUserFields && user) {
+        if (shouldUpdateUserFields) {
           row[key] = [user.userId]
         }
         break
@@ -98,6 +186,9 @@ export function processAutoColumn(
   }
   return { table, row }
 }
+exports.processAutoColumn = processAutoColumn
+exports.fixAutoColumnSubType = fixAutoColumnSubType
+exports.processFormulas = processFormulas
 
 /**
  * This will coerce a value to the correct types based on the type transform map
@@ -105,17 +196,15 @@ export function processAutoColumn(
  * @param {object} type The type fo coerce to
  * @returns {object} The coerced value
  */
-export function coerce(row: any, type: any) {
+exports.coerce = (row, type) => {
   // no coercion specified for type, skip it
   if (!TYPE_TRANSFORM_MAP[type]) {
     return row
   }
   // eslint-disable-next-line no-prototype-builtins
   if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
-    // @ts-ignore
     return TYPE_TRANSFORM_MAP[type][row]
   } else if (TYPE_TRANSFORM_MAP[type].parse) {
-    // @ts-ignore
     return TYPE_TRANSFORM_MAP[type].parse(row)
   }
 
@@ -131,12 +220,12 @@ export function coerce(row: any, type: any) {
  * @param {object} opts some input processing options (like disabling auto-column relationships).
  * @returns {object} the row which has been prepared to be written to the DB.
  */
-export function inputProcessing(
-  user: ContextUser,
-  table: Table,
-  row: Row,
-  opts?: AutoColumnProcessingOpts
-) {
+exports.inputProcessing = (
+  user = {},
+  table,
+  row,
+  opts = { noAutoRelationships: false }
+) => {
   let clonedRow = cloneDeep(row)
   // need to copy the table so it can be differenced on way out
   const copiedTable = cloneDeep(table)
@@ -156,7 +245,7 @@ export function inputProcessing(
     }
     // otherwise coerce what is there to correct types
     else {
-      clonedRow[key] = coerce(value, field.type)
+      clonedRow[key] = exports.coerce(value, field.type)
     }
   }
 
@@ -178,21 +267,17 @@ export function inputProcessing(
  * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
  * @returns {object[]|object} the enriched rows will be returned.
  */
-export async function outputProcessing(
-  table: Table,
-  rows: Row[] | Row,
-  opts = { squash: true }
-) {
+exports.outputProcessing = async (table, rows, opts = { squash: true }) => {
   let wasArray = true
   if (!(rows instanceof Array)) {
     rows = [rows]
     wasArray = false
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(table, rows as Row[])
+  let enriched = await linkRows.attachFullLinkedDocs(table, rows)
 
   // process formulas
-  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
+  enriched = processFormulas(table, enriched, { dynamic: true })
 
   // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
@@ -201,7 +286,7 @@ export async function outputProcessing(
         if (row[property] == null || !Array.isArray(row[property])) {
           continue
         }
-        row[property].forEach((attachment: any) => {
+        row[property].forEach(attachment => {
           attachment.url = attachmentsRelativeURL(attachment.key)
         })
       }
@@ -223,28 +308,20 @@ export async function outputProcessing(
  * deleted attachment columns.
  * @return {Promise<void>} When all attachments have been removed this will return.
  */
-export async function cleanupAttachments(
-  table: Table,
-  {
-    row,
-    rows,
-    oldRow,
-    oldTable,
-  }: { row?: Row; rows?: Row[]; oldRow?: Row; oldTable?: Table }
-): Promise<any> {
-  const appId = context.getAppId()
-  if (!dbCore.isProdAppID(appId)) {
-    const prodAppId = dbCore.getProdAppID(appId!)
+exports.cleanupAttachments = async (table, { row, rows, oldRow, oldTable }) => {
+  const appId = getAppId()
+  if (!isProdAppID(appId)) {
+    const prodAppId = getProdAppID(appId)
     // if prod exists, then don't allow deleting
-    const exists = await dbCore.dbExists(prodAppId)
+    const exists = await dbExists(prodAppId)
     if (exists) {
       return
     }
   }
-  let files: string[] = []
-  function addFiles(row: Row, key: string) {
+  let files = []
+  function addFiles(row, key) {
     if (row[key]) {
-      files = files.concat(row[key].map((attachment: any) => attachment.key))
+      files = files.concat(row[key].map(attachment => attachment.key))
     }
   }
   const schemaToUse = oldTable ? oldTable.schema : table.schema
@@ -253,7 +330,7 @@ export async function cleanupAttachments(
       continue
     }
     // old table had this column, new table doesn't - delete it
-    if (rows && oldTable && !table.schema[key]) {
+    if (oldTable && !table.schema[key]) {
       rows.forEach(row => addFiles(row, key))
     } else if (oldRow && row) {
       // if updating, need to manage the differences
@@ -265,6 +342,6 @@ export async function cleanupAttachments(
     }
   }
   if (files.length > 0) {
-    return objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
+    return deleteFiles(ObjectStoreBuckets.APPS, files)
   }
 }

package/src/utilities/rowProcessor/{utils.ts → utils.js}

@@ -1,17 +1,16 @@
-import {
+const {
   FieldTypes,
   FormulaTypes,
   AutoFieldDefaultNames,
   AutoFieldSubTypes,
-} from "../../constants"
-import { processStringSync } from "@budibase/string-templates"
-import { FieldSchema, Table, Row } from "@budibase/types"
+} = require("../../constants")
+const { processStringSync } = require("@budibase/string-templates")
 
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
  */
-export function fixAutoColumnSubType(column: FieldSchema) {
+exports.fixAutoColumnSubType = column => {
   if (!column.autocolumn || !column.name || column.subtype) {
     return column
   }
@@ -33,18 +32,15 @@ export function fixAutoColumnSubType(column: FieldSchema) {
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-export function processFormulas(
-  table: Table,
-  rows: Row[] | Row,
-  { dynamic, contextRows }: any = { dynamic: true }
-) {
+exports.processFormulas = (
+  table,
+  rows,
+  { dynamic, contextRows } = { dynamic: true }
+) => {
   const single = !Array.isArray(rows)
-  let rowArray: Row[]
   if (single) {
-    rowArray = [rows]
+    rows = [rows]
     contextRows = contextRows ? [contextRows] : contextRows
-  } else {
-    rowArray = rows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
     const isStatic = schema.formulaType === FormulaTypes.STATIC
@@ -56,25 +52,25 @@ export function processFormulas(
       continue
     }
     // iterate through rows and process formula
-    for (let i = 0; i < rowArray.length; i++) {
+    for (let i = 0; i < rows.length; i++) {
       if (schema.formula) {
-        let row = rowArray[i]
+        let row = rows[i]
         let context = contextRows ? contextRows[i] : row
-        rowArray[i] = {
+        rows[i] = {
           ...row,
           [column]: processStringSync(schema.formula, context),
         }
       }
     }
   }
-  return single ? rowArray[0] : rowArray
+  return single ? rows[0] : rows
 }
 
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.
  */
-export function processDates(table: Table, rows: Row[]) {
+exports.processDates = (table, rows) => {
   let datesWithTZ = []
   for (let [column, schema] of Object.entries(table.schema)) {
     if (schema.type !== FieldTypes.DATETIME) {

package/src/utilities/{scriptRunner.ts → scriptRunner.js}

@@ -1,13 +1,9 @@
-import fetch from "node-fetch"
-import { VM, VMScript } from "vm2"
+const fetch = require("node-fetch")
+const { VM, VMScript } = require("vm2")
 const JS_TIMEOUT_MS = 1000
 
 class ScriptRunner {
-  vm: VM
-  results: { out: string }
-  script: VMScript
-
-  constructor(script: string, context: any) {
+  constructor(script, context) {
     const code = `let fn = () => {\n${script}\n}; results.out = fn();`
     this.vm = new VM({
       timeout: JS_TIMEOUT_MS,
@@ -25,4 +21,4 @@ class ScriptRunner {
   }
 }
 
-export = ScriptRunner
+module.exports = ScriptRunner