@budibase/server 2.0.33 → 2.0.34-alpha.0

package/src/automations/bullboard.js

@@ -1,37 +1,17 @@
 const { createBullBoard } = require("@bull-board/api")
 const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
-const env = require("../environment")
-const Queue = env.isTest()
-  ? require("../utilities/queue/inMemoryQueue")
-  : require("bull")
-const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/backend-core/redis")
-const { opts, redisProtocolUrl } = utils.getRedisOptions()
-const listeners = require("./listeners")
+const { queue } = require("@budibase/backend-core")
+const automation = require("../threads/automation")
 
-const CLEANUP_PERIOD_MS = 60 * 1000
-const queueConfig = redisProtocolUrl || { redis: opts }
-let cleanupInternal = null
-
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
-listeners.addListeners(automationQueue)
-
-async function cleanup() {
-  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
-}
+let automationQueue = queue.createQueue(
+  queue.JobQueue.AUTOMATION,
+  automation.removeStalled
+)
 
 const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  // cleanup the events every 5 minutes
-  if (!cleanupInternal) {
-    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
-    // fire off an initial cleanup
-    cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
-    })
-  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
@@ -48,12 +28,7 @@ exports.init = () => {
 }
 
 exports.shutdown = async () => {
-  if (automationQueue) {
-    clearInterval(cleanupInternal)
-    await automationQueue.close()
-    automationQueue = null
-  }
-  console.log("Bull shutdown")
+  await queue.shutdown()
 }
 
-exports.queue = automationQueue
+exports.automationQueue = automationQueue

package/src/automations/index.js

@@ -1,5 +1,5 @@
 const { processEvent } = require("./utils")
-const { queue, shutdown } = require("./bullboard")
+const { automationQueue, shutdown } = require("./bullboard")
 const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
@@ -8,7 +8,7 @@ const { ACTION_DEFINITIONS } = require("./actions")
  */
 exports.init = async function () {
   // this promise will not complete
-  const promise = queue.process(async job => {
+  const promise = automationQueue.process(async job => {
     await processEvent(job)
   })
   // on init we need to trigger any reboot automations
@@ -17,13 +17,13 @@ exports.init = async function () {
 }
 
 exports.getQueues = () => {
-  return [queue]
+  return [automationQueue]
 }
 
 exports.shutdown = () => {
   return shutdown()
 }
 
-exports.queue = queue
+exports.automationQueue = automationQueue
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS
 exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS

package/src/automations/steps/updateRow.js

@@ -66,9 +66,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
   }
   const tableId = inputs.row.tableId
 
-  // clear any falsy properties so that they aren't updated
+  // clear any undefined, null or empty string properties so that they aren't updated
   for (let propKey of Object.keys(inputs.row)) {
-    if (!inputs.row[propKey] || inputs.row[propKey] === "") {
+    if (inputs.row[propKey] == null || inputs.row[propKey] === "") {
       delete inputs.row[propKey]
     }
   }

package/src/automations/tests/automation.spec.js

@@ -29,16 +29,11 @@ describe("Run through some parts of the automations system", () => {
   afterAll(setup.afterAll)
 
   it("should be able to init in builder", async () => {
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
+    await triggers.externalTrigger(basicAutomation(), { a: 1, appId: "app_123" })
     await wait(100)
     expect(thread.execute).toHaveBeenCalled()
   })
 
-  it("should be able to init in prod", async () => {
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await wait(100)
-  })
-
   it("should check coercion", async () => {
     const table = await config.createTable()
     const automation = basicAutomation()

package/src/automations/triggers.js

@@ -4,7 +4,7 @@ const { coerce } = require("../utilities/rowProcessor")
 const { definitions } = require("./triggerInfo")
 const { isDevAppID } = require("../db/utils")
 // need this to call directly, so we can get a response
-const { queue } = require("./bullboard")
+const { automationQueue } = require("./bullboard")
 const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
@@ -56,7 +56,7 @@ async function queueRelevantRowAutomations(event, eventType) {
         automationTrigger.inputs &&
         automationTrigger.inputs.tableId === event.row.tableId
       ) {
-        await queue.add({ automation, event }, JOB_OPTS)
+        await automationQueue.add({ automation, event }, JOB_OPTS)
       }
     }
   })
@@ -110,7 +110,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data, JOB_OPTS)
+    return automationQueue.add(data, JOB_OPTS)
   }
 }
 
@@ -136,7 +136,7 @@ exports.rebootTrigger = async () => {
               timestamp: Date.now(),
             },
           }
-          rebootEvents.push(queue.add(job, JOB_OPTS))
+          rebootEvents.push(automationQueue.add(job, JOB_OPTS))
         }
       }
       await Promise.all(rebootEvents)

package/src/automations/utils.ts

@@ -1,7 +1,7 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
 import * as webhooks from "../api/controllers/webhook"
-import { queue } from "./bullboard"
+import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
 import { MetadataTypes, WebhookType } from "../constants"
@@ -13,7 +13,7 @@ import {
   getAppId,
   getProdAppDB,
 } from "@budibase/backend-core/context"
-import { tenancy } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { Automation } from "@budibase/types"
 
@@ -28,12 +28,14 @@ const jobMessage = (job: any, message: string) => {
 
 export async function processEvent(job: any) {
   try {
+    const automationId = job.data.automation._id
     console.log(jobMessage(job, "running"))
     // need to actually await these so that an error can be captured properly
-    const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
-    return await tenancy.doInTenant(tenantId, async () => {
+    return await context.doInContext(job.data.event.appId, async () => {
       const runFn = () => Runner.run(job)
-      return quotas.addAutomation(runFn)
+      return quotas.addAutomation(runFn, {
+        automationId,
+      })
     })
   } catch (err) {
     const errJson = JSON.stringify(err)
@@ -77,21 +79,25 @@ export function removeDeprecated(definitions: any) {
 // end the repetition and the job itself
 export async function disableAllCrons(appId: any) {
   const promises = []
-  const jobs = await queue.getRepeatableJobs()
+  const jobs = await automationQueue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
-      promises.push(queue.removeRepeatableByKey(job.key))
+      promises.push(automationQueue.removeRepeatableByKey(job.key))
       if (job.id) {
-        promises.push(queue.removeJobs(job.id))
+        promises.push(automationQueue.removeJobs(job.id))
       }
     }
   }
   return Promise.all(promises)
 }
 
-export async function disableCron(jobId: string, jobKey: string) {
-  await queue.removeRepeatableByKey(jobKey)
-  await queue.removeJobs(jobId)
+export async function disableCronById(jobId: number | string) {
+  const repeatJobs = await automationQueue.getRepeatableJobs()
+  for (let repeatJob of repeatJobs) {
+    if (repeatJob.id === jobId) {
+      await automationQueue.removeRepeatableByKey(repeatJob.key)
+    }
+  }
   console.log(`jobId=${jobId} disabled`)
 }
 
@@ -139,7 +145,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
   ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job: any = await queue.add(
+    const job: any = await automationQueue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },

package/src/constants/index.js

@@ -1,10 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { UserStatus } = require("@budibase/backend-core/constants")
-const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore")
-
-exports.JobQueues = {
-  AUTOMATIONS: "automationQueue",
-}
+const { objectStore } = require("@budibase/backend-core")
 
 const FilterTypes = {
   STRING: "string",
@@ -31,6 +27,7 @@ exports.NoEmptyFilterStrings = [
 
 exports.FieldTypes = {
   STRING: "string",
+  BARCODEQR: "barcodeqr",
   LONGFORM: "longform",
   OPTIONS: "options",
   NUMBER: "number",
@@ -51,6 +48,7 @@ exports.CanSwitchTypes = [
     exports.FieldTypes.STRING,
     exports.FieldTypes.OPTIONS,
     exports.FieldTypes.LONGFORM,
+    exports.FieldTypes.BARCODEQR,
   ],
   [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
 ]
@@ -209,6 +207,6 @@ exports.AutomationErrors = {
 }
 
 // pass through the list from the auth/core lib
-exports.ObjectStoreBuckets = ObjectStoreBuckets
+exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 
 exports.MAX_AUTOMATION_RECURRING_ERRORS = 5

package/src/db/utils.js

@@ -1,6 +1,7 @@
 const newid = require("./newid")
 const {
-  DocumentType: CoreDocTypes,
+  DocumentType: CoreDocType,
+  InternalTable,
   getRoleParams,
   generateRoleID,
   APP_DEV_PREFIX,
@@ -13,6 +14,12 @@ const {
   generateAppID,
   getQueryIndex,
   ViewName,
+  getDocParams,
+  getRowParams,
+  generateRowID,
+  getUserMetadataParams,
+  generateUserMetadataID,
+  getGlobalIDFromUserMetadataID,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -23,30 +30,7 @@ const AppStatus = {
   DEPLOYED: "published",
 }
 
-const DocumentType = {
-  ...CoreDocTypes,
-  TABLE: "ta",
-  ROW: "ro",
-  USER: "us",
-  AUTOMATION: "au",
-  LINK: "li",
-  WEBHOOK: "wh",
-  INSTANCE: "inst",
-  LAYOUT: "layout",
-  SCREEN: "screen",
-  DATASOURCE: "datasource",
-  DATASOURCE_PLUS: "datasource_plus",
-  QUERY: "query",
-  DEPLOYMENTS: "deployments",
-  METADATA: "metadata",
-  MEM_VIEW: "view",
-  USER_FLAG: "flag",
-  AUTOMATION_METADATA: "meta_au",
-}
-
-const InternalTables = {
-  USER_METADATA: "ta_users",
-}
+const DocumentType = CoreDocType
 
 const SearchIndexes = {
   ROWS: "rows",
@@ -66,11 +50,11 @@ exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
 exports.isDevAppID = isDevAppID
 exports.isProdAppID = isProdAppID
-exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
 exports.TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 exports.ViewName = ViewName
-exports.InternalTables = InternalTables
+exports.InternalTables = InternalTable
 exports.DocumentType = DocumentType
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
@@ -79,36 +63,15 @@ exports.AppStatus = AppStatus
 exports.BudibaseInternalDB = BudibaseInternalDB
 exports.generateAppID = generateAppID
 exports.generateDevAppID = getDevelopmentAppID
-
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
-
 exports.getQueryIndex = getQueryIndex
-
-/**
- * If creating DB allDocs/query params with only a single top level ID this can be used, this
- * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
- * More complex cases such as link docs and rows which have multiple levels of IDs that their
- * ID consists of need their own functions to build the allDocs parameters.
- * @param {string} docType The type of document which input params are being built for, e.g. user,
- * link, app, table and so on.
- * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
- * for a singular document.
- * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
- * @returns {object} Parameters which can then be used with an allDocs request.
- */
-function getDocParams(docType, docId = null, otherProps = {}) {
-  if (docId == null) {
-    docId = ""
-  }
-  return {
-    ...otherProps,
-    startkey: `${docType}${SEPARATOR}${docId}`,
-    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
-  }
-}
-
 exports.getDocParams = getDocParams
+exports.getRowParams = getRowParams
+exports.generateRowID = generateRowID
+exports.getUserMetadataParams = getUserMetadataParams
+exports.generateUserMetadataID = generateUserMetadataID
+exports.getGlobalIDFromUserMetadataID = getGlobalIDFromUserMetadataID
 
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
@@ -125,24 +88,6 @@ exports.generateTableID = () => {
   return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
 }
 
-/**
- * Gets the DB allDocs/query params for retrieving a row.
- * @param {string|null} tableId The table in which the rows have been stored.
- * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
- * left null to get all the rows in the table.
- * @param {object} otherProps Any other properties to add to the request.
- * @returns {object} Parameters which can then be used with an allDocs request.
- */
-exports.getRowParams = (tableId = null, rowId = null, otherProps = {}) => {
-  if (tableId == null) {
-    return getDocParams(DocumentType.ROW, null, otherProps)
-  }
-
-  const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId
-
-  return getDocParams(DocumentType.ROW, endOfKey, otherProps)
-}
-
 /**
  * Given a row ID this will find the table ID within it (only works for internal tables).
  * @param {string} rowId The ID of the row.
@@ -155,44 +100,6 @@ exports.getTableIDFromRowID = rowId => {
   return `${DocumentType.TABLE}${SEPARATOR}${components[0]}`
 }
 
-/**
- * Gets a new row ID for the specified table.
- * @param {string} tableId The table which the row is being created for.
- * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
- * @returns {string} The new ID which a row doc can be stored under.
- */
-exports.generateRowID = (tableId, id = null) => {
-  id = id || newid()
-  return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}`
-}
-
-/**
- * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
- */
-exports.getUserMetadataParams = (userId = null, otherProps = {}) => {
-  return exports.getRowParams(InternalTables.USER_METADATA, userId, otherProps)
-}
-
-/**
- * Generates a new user ID based on the passed in global ID.
- * @param {string} globalId The ID of the global user.
- * @returns {string} The new user ID which the user doc can be stored under.
- */
-exports.generateUserMetadataID = globalId => {
-  return exports.generateRowID(InternalTables.USER_METADATA, globalId)
-}
-
-/**
- * Breaks up the ID to get the global ID.
- */
-exports.getGlobalIDFromUserMetadataID = id => {
-  const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-  if (!id || !id.includes(prefix)) {
-    return id
-  }
-  return id.split(prefix)[1]
-}
-
 /**
  * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
  */

package/src/definitions/automations.ts

@@ -27,18 +27,6 @@ export interface TriggerOutput {
   timestamp?: number
 }
 
-export interface AutomationEvent {
-  data: {
-    automation: Automation
-    event: any
-  }
-  opts?: {
-    repeat?: {
-      jobId: string
-    }
-  }
-}
-
 export interface AutomationContext extends AutomationResults {
   steps: any[]
   trigger: any

package/src/definitions/datasource.ts

@@ -31,6 +31,7 @@ export interface BearerAuthConfig {
 
 export interface RestConfig {
   url: string
+  rejectUnauthorized: boolean
   defaultHeaders: {
     [key: string]: any
   }

package/src/index.ts

@@ -8,6 +8,7 @@ function runServer() {
   checkDevelopmentEnvironment()
   fixPath()
   // this will setup http and https proxies form env variables
+  process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = "false"
   bootstrap()
   require("./app")
 }

package/src/integrations/arangodb.ts

@@ -62,6 +62,8 @@ class ArangoDBIntegration implements IntegrationBase {
 
   constructor(config: ArangodbConfig) {
     const newConfig = {
+      url: config.url,
+      databaseName: config.databaseName,
       auth: {
         username: config.username,
         password: config.password,

package/src/integrations/base/sqlTable.ts

@@ -40,6 +40,7 @@ function generateSchema(
       case FieldTypes.STRING:
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
+      case FieldTypes.BARCODEQR:
         schema.text(key)
         break
       case FieldTypes.NUMBER: