@budibase/server 2.0.38 → 2.0.40-alpha.0

package/src/api/routes/index.ts

@@ -25,11 +25,17 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
+import Router from "@koa/router"
+import { api } from "@budibase/pro"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
 
-export const mainRoutes = [
+const appBackupRoutes = api.appBackups
+const scheduleRoutes = api.schedules
+export const mainRoutes: Router[] = [
+  appBackupRoutes,
+  backupRoutes,
   authRoutes,
   deployRoutes,
   layoutRoutes,
@@ -49,14 +55,14 @@ export const mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  backupRoutes,
   metadataRoutes,
   devRoutes,
   cloudRoutes,
-  // these need to be handled last as they still use /api/:tableId
-  // this could be breaking as koa may recognise other routes as this
-  tableRoutes,
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
+  scheduleRoutes,
+  // these need to be handled last as they still use /api/:tableId
+  // this could be breaking as koa may recognise other routes as this
+  tableRoutes,
 ]

package/src/api/routes/tests/backup.spec.js

@@ -21,7 +21,7 @@ describe("/backups", () => {
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.text).toBeDefined()
-      expect(res.text.includes(`"db_name":"${config.getAppId()}"`)).toEqual(true)
+      expect(res.headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported.mock.calls.length).toBe(1)
     })
 

package/src/api/routes/tests/{deployment.spec.js → deployment.spec.ts}

@@ -1,5 +1,5 @@
-const setup = require("./utilities")
-const { events } = require("@budibase/backend-core")
+import setup from "./utilities"
+import { events } from "@budibase/backend-core"
 
 describe("/deployments", () => {
   let request = setup.getRequest()
@@ -19,7 +19,7 @@ describe("/deployments", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(events.app.published.mock.calls.length).toBe(1)
+      expect((events.app.published as jest.Mock).mock.calls.length).toBe(1)
     })
   })
-})
+})

package/src/api/routes/tests/row.spec.js

@@ -34,18 +34,13 @@ describe("/rows", () => {
       .expect(status)
 
   const getRowUsage = async () => {
-    return config.doInContext(null, () =>
-      quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
-    )
+    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
+    return total
   }
 
   const getQueryUsage = async () => {
-    return config.doInContext(null, () =>
-      quotas.getCurrentUsageValue(
-        QuotaUsageType.MONTHLY,
-        MonthlyQuotaName.QUERIES
-      )
-    )
+    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
+    return total
   }
 
   const assertRowUsage = async expected => {
@@ -60,26 +55,26 @@ describe("/rows", () => {
 
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
-      // const rowUsage = await getRowUsage()
-      // const queryUsage = await getQueryUsage()
-      //
-      // const res = await request
-      //   .post(`/api/${row.tableId}/rows`)
-      //   .send(row)
-      //   .set(config.defaultHeaders())
-      //   .expect('Content-Type', /json/)
-      //   .expect(200)
-      // expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
-      // expect(res.body.name).toEqual("Test Contact")
-      // expect(res.body._rev).toBeDefined()
-      // await assertRowUsage(rowUsage + 1)
-      // await assertQueryUsage(queryUsage + 1)
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
+      const res = await request
+        .post(`/api/${row.tableId}/rows`)
+        .send(row)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+      expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
+      expect(res.body.name).toEqual("Test Contact")
+      expect(res.body._rev).toBeDefined()
+      await assertRowUsage(rowUsage + 1)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
-      // const rowUsage = await getRowUsage()
-      // const queryUsage = await getQueryUsage()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${table._id}/rows`)
@@ -97,8 +92,8 @@ describe("/rows", () => {
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
-      // await assertRowUsage(rowUsage)
-      // await assertQueryUsage(queryUsage + 1)
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should load a row", async () => {

package/src/api/routes/utils/validators.js

@@ -1,10 +1,10 @@
 const { joiValidator } = require("@budibase/backend-core/auth")
 const { DataSourceOperation } = require("../../../constants")
-const { WebhookType } = require("../../../constants")
 const {
   BUILTIN_PERMISSION_IDS,
   PermissionLevels,
 } = require("@budibase/backend-core/permissions")
+const { WebhookActionType } = require("@budibase/types")
 const Joi = require("joi")
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
@@ -126,7 +126,7 @@ exports.webhookValidator = () => {
     name: Joi.string().required(),
     bodySchema: Joi.object().optional(),
     action: Joi.object({
-      type: Joi.string().required().valid(WebhookType.AUTOMATION),
+      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
       target: Joi.string().required(),
     }).required(),
   }).unknown(true))

package/src/api/routes/{webhook.js → webhook.ts}

@@ -1,9 +1,10 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/webhook")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/backend-core/permissions")
-const { webhookValidator } = require("./utils/validators")
+import Router from "@koa/router"
+import * as controller from "../controllers/webhook"
+import authorized from "../../middleware/authorized"
+import { permissions } from "@budibase/backend-core"
+import { webhookValidator } from "./utils/validators"
 
+const BUILDER = permissions.BUILDER
 const router = new Router()
 
 router
@@ -23,4 +24,4 @@ router
   // this shouldn't have authorisation, right now its always public
   .post("/api/webhooks/trigger/:instance/:id", controller.trigger)
 
-module.exports = router
+export default router

package/src/app.ts

@@ -15,28 +15,16 @@ db.init()
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")
-const pino = require("koa-pino-logger")
 const http = require("http")
 const api = require("./api")
-const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
-const fileSystem = require("./utilities/fileSystem")
-const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
-const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
-const fs = require("fs")
 import redis from "./utilities/redis"
-import * as migrations from "./migrations"
-import { events, installation, tenancy } from "@budibase/backend-core"
-import {
-  createAdminUser,
-  generateApiKey,
-  getChecklist,
-} from "./utilities/workerRequests"
-import { watch } from "./watch"
+import { events } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
+import { startup } from "./startup"
 
 const app = new Koa()
 
@@ -52,19 +40,6 @@ app.use(
   })
 )
 
-app.use(pino(pinoSettings()))
-
-if (!env.isTest()) {
-  const plugin = bullboard.init()
-  app.use(plugin)
-}
-
-app.context.eventEmitter = eventEmitter
-app.context.auth = {}
-
-// api routes
-app.use(api.router.routes())
-
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
   Sentry.init()
@@ -103,70 +78,7 @@ server.on("close", async () => {
 })
 
 module.exports = server.listen(env.PORT || 0, async () => {
-  console.log(`Budibase running on ${JSON.stringify(server.address())}`)
-  env._set("PORT", server.address().port)
-  eventEmitter.emitPort(env.PORT)
-  fileSystem.init()
-  await redis.init()
-
-  // run migrations on startup if not done via http
-  // not recommended in a clustered environment
-  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
-    try {
-      await migrations.migrate()
-    } catch (e) {
-      logAlert("Error performing migrations. Exiting.", e)
-      shutdown()
-    }
-  }
-
-  // check and create admin user if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.BB_ADMIN_USER_EMAIL &&
-    env.BB_ADMIN_USER_PASSWORD
-  ) {
-    const checklist = await getChecklist()
-    if (!checklist?.adminUser?.checked) {
-      try {
-        const tenantId = tenancy.getTenantId()
-        const user = await createAdminUser(
-          env.BB_ADMIN_USER_EMAIL,
-          env.BB_ADMIN_USER_PASSWORD,
-          tenantId
-        )
-        // Need to set up an API key for automated integration tests
-        if (env.isTest()) {
-          await generateApiKey(user._id)
-        }
-
-        console.log(
-          "Admin account automatically created for",
-          env.BB_ADMIN_USER_EMAIL
-        )
-      } catch (e) {
-        logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown()
-      }
-    }
-  }
-
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // done last - this will never complete
-  await automations.init()
+  await startup(app, server)
 })
 
 const shutdown = () => {

package/src/automations/automationUtils.js

@@ -1,9 +1,9 @@
-const { getTable } = require("../api/controllers/table/utils")
 const {
   decodeJSBinding,
   isJSBinding,
   encodeJSBinding,
 } = require("@budibase/string-templates")
+const sdk = require("../sdk")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -63,7 +63,7 @@ exports.cleanInputValues = (inputs, schema) => {
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
 exports.cleanUpRow = async (tableId, row) => {
-  let table = await getTable(tableId)
+  let table = await sdk.tables.getTable(tableId)
   return exports.cleanInputValues(row, { properties: table.schema })
 }
 

package/src/automations/bullboard.js

@@ -1,39 +1,24 @@
 const { createBullBoard } = require("@bull-board/api")
 const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
-const env = require("../environment")
-const Queue = env.isTest()
-  ? require("../utilities/queue/inMemoryQueue")
-  : require("bull")
-const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/backend-core/redis")
-const { opts, redisProtocolUrl } = utils.getRedisOptions()
-const listeners = require("./listeners")
+const { queue } = require("@budibase/backend-core")
+const automation = require("../threads/automation")
+const { backups } = require("@budibase/pro")
 
-const CLEANUP_PERIOD_MS = 60 * 1000
-const queueConfig = redisProtocolUrl || { redis: opts }
-let cleanupInternal = null
-
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
-listeners.addListeners(automationQueue)
-
-async function cleanup() {
-  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
-}
+let automationQueue = queue.createQueue(
+  queue.JobQueue.AUTOMATION,
+  automation.removeStalled
+)
 
 const PATH_PREFIX = "/bulladmin"
 
-exports.init = () => {
-  // cleanup the events every 5 minutes
-  if (!cleanupInternal) {
-    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
-    // fire off an initial cleanup
-    cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
-    })
-  }
+exports.init = async () => {
   // Set up queues for bull board admin
+  const backupQueue = await backups.getBackupQueue()
   const queues = [automationQueue]
+  if (backupQueue) {
+    queues.push(backupQueue)
+  }
   const adapters = []
   const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
@@ -48,12 +33,7 @@ exports.init = () => {
 }
 
 exports.shutdown = async () => {
-  if (automationQueue) {
-    clearInterval(cleanupInternal)
-    await automationQueue.close()
-    automationQueue = null
-  }
-  console.log("Bull shutdown")
+  await queue.shutdown()
 }
 
-exports.queue = automationQueue
+exports.automationQueue = automationQueue

package/src/automations/index.js

@@ -1,5 +1,5 @@
 const { processEvent } = require("./utils")
-const { queue, shutdown } = require("./bullboard")
+const { automationQueue, shutdown } = require("./bullboard")
 const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
@@ -8,7 +8,7 @@ const { ACTION_DEFINITIONS } = require("./actions")
  */
 exports.init = async function () {
   // this promise will not complete
-  const promise = queue.process(async job => {
+  const promise = automationQueue.process(async job => {
     await processEvent(job)
   })
   // on init we need to trigger any reboot automations
@@ -17,13 +17,13 @@ exports.init = async function () {
 }
 
 exports.getQueues = () => {
-  return [queue]
+  return [automationQueue]
 }
 
 exports.shutdown = () => {
   return shutdown()
 }
 
-exports.queue = queue
+exports.automationQueue = automationQueue
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS
 exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS

package/src/automations/steps/updateRow.js

@@ -66,9 +66,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
   }
   const tableId = inputs.row.tableId
 
-  // clear any falsy properties so that they aren't updated
+  // clear any undefined, null or empty string properties so that they aren't updated
   for (let propKey of Object.keys(inputs.row)) {
-    if (!inputs.row[propKey] || inputs.row[propKey] === "") {
+    if (inputs.row[propKey] == null || inputs.row[propKey] === "") {
       delete inputs.row[propKey]
     }
   }

package/src/automations/tests/automation.spec.js

@@ -29,16 +29,11 @@ describe("Run through some parts of the automations system", () => {
   afterAll(setup.afterAll)
 
   it("should be able to init in builder", async () => {
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
+    await triggers.externalTrigger(basicAutomation(), { a: 1, appId: "app_123" })
     await wait(100)
     expect(thread.execute).toHaveBeenCalled()
   })
 
-  it("should be able to init in prod", async () => {
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await wait(100)
-  })
-
   it("should check coercion", async () => {
     const table = await config.createTable()
     const automation = basicAutomation()

package/src/automations/triggers.js

@@ -4,7 +4,7 @@ const { coerce } = require("../utilities/rowProcessor")
 const { definitions } = require("./triggerInfo")
 const { isDevAppID } = require("../db/utils")
 // need this to call directly, so we can get a response
-const { queue } = require("./bullboard")
+const { automationQueue } = require("./bullboard")
 const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
@@ -56,7 +56,7 @@ async function queueRelevantRowAutomations(event, eventType) {
         automationTrigger.inputs &&
         automationTrigger.inputs.tableId === event.row.tableId
       ) {
-        await queue.add({ automation, event }, JOB_OPTS)
+        await automationQueue.add({ automation, event }, JOB_OPTS)
       }
     }
   })
@@ -110,14 +110,14 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data, JOB_OPTS)
+    return automationQueue.add(data, JOB_OPTS)
   }
 }
 
 exports.rebootTrigger = async () => {
   // reboot cron option is only available on the main thread at
-  // startup and only usable in self host
-  if (env.isInThread() || !env.SELF_HOSTED) {
+  // startup and only usable in self host and single tenant environments
+  if (env.isInThread() || !env.SELF_HOSTED || env.MULTI_TENANCY) {
     return
   }
   // iterate through all production apps, find the reboot crons
@@ -136,7 +136,7 @@ exports.rebootTrigger = async () => {
               timestamp: Date.now(),
             },
           }
-          rebootEvents.push(queue.add(job, JOB_OPTS))
+          rebootEvents.push(automationQueue.add(job, JOB_OPTS))
         }
       }
       await Promise.all(rebootEvents)

package/src/automations/utils.ts

@@ -1,10 +1,9 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
-import * as webhooks from "../api/controllers/webhook"
-import { queue } from "./bullboard"
+import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
-import { MetadataTypes, WebhookType } from "../constants"
+import { MetadataTypes } from "../constants"
 import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
@@ -13,9 +12,10 @@ import {
   getAppId,
   getProdAppDB,
 } from "@budibase/backend-core/context"
-import { tenancy } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
-import { Automation } from "@budibase/types"
+import { Automation, WebhookActionType } from "@budibase/types"
+import sdk from "../sdk"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -28,12 +28,14 @@ const jobMessage = (job: any, message: string) => {
 
 export async function processEvent(job: any) {
   try {
+    const automationId = job.data.automation._id
     console.log(jobMessage(job, "running"))
     // need to actually await these so that an error can be captured properly
-    const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
-    return await tenancy.doInTenant(tenantId, async () => {
+    return await context.doInContext(job.data.event.appId, async () => {
       const runFn = () => Runner.run(job)
-      return quotas.addAutomation(runFn)
+      return quotas.addAutomation(runFn, {
+        automationId,
+      })
     })
   } catch (err) {
     const errJson = JSON.stringify(err)
@@ -77,21 +79,25 @@ export function removeDeprecated(definitions: any) {
 // end the repetition and the job itself
 export async function disableAllCrons(appId: any) {
   const promises = []
-  const jobs = await queue.getRepeatableJobs()
+  const jobs = await automationQueue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
-      promises.push(queue.removeRepeatableByKey(job.key))
+      promises.push(automationQueue.removeRepeatableByKey(job.key))
       if (job.id) {
-        promises.push(queue.removeJobs(job.id))
+        promises.push(automationQueue.removeJobs(job.id))
       }
     }
   }
   return Promise.all(promises)
 }
 
-export async function disableCron(jobId: string, jobKey: string) {
-  await queue.removeRepeatableByKey(jobKey)
-  await queue.removeJobs(jobId)
+export async function disableCronById(jobId: number | string) {
+  const repeatJobs = await automationQueue.getRepeatableJobs()
+  for (let repeatJob of repeatJobs) {
+    if (repeatJob.id === jobId) {
+      await automationQueue.removeRepeatableByKey(repeatJob.key)
+    }
+  }
   console.log(`jobId=${jobId} disabled`)
 }
 
@@ -139,7 +145,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
   ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job: any = await queue.add(
+    const job: any = await automationQueue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },
@@ -191,16 +197,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
       let db = getAppDB()
       // need to get the webhook to get the rev
       const webhook = await db.get(oldTrigger.webhookId)
-      const ctx = {
-        appId,
-        params: { id: webhook._id, rev: webhook._rev },
-      }
       // might be updating - reset the inputs to remove the URLs
       if (newTrigger) {
         delete newTrigger.webhookId
         newTrigger.inputs = {}
       }
-      await webhooks.destroy(ctx)
+      await sdk.automations.webhook.destroy(webhook._id, webhook._rev)
     } catch (err) {
       // don't worry about not being able to delete, if it doesn't exist all good
     }
@@ -210,18 +212,14 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx: any = {
-      appId,
-      request: {
-        body: new webhooks.Webhook(
-          "Automation webhook",
-          WebhookType.AUTOMATION,
-          newAuto._id
-        ),
-      },
-    }
-    await webhooks.save(ctx)
-    const id = ctx.body.webhook._id
+    const webhook = await sdk.automations.webhook.save(
+      sdk.automations.webhook.newDoc(
+        "Automation webhook",
+        WebhookActionType.AUTOMATION,
+        newAuto._id
+      )
+    )
+    const id = webhook._id
     newTrigger.webhookId = id
     // the app ID has to be development for this endpoint
     // it can only be used when building the app