@budibase/server 2.0.34-alpha.5 → 2.0.34-alpha.7

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.0.34-alpha.5",
+  "version": "2.0.34-alpha.7",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.0.34-alpha.5",
-    "@budibase/client": "2.0.34-alpha.5",
-    "@budibase/pro": "2.0.34-alpha.4",
-    "@budibase/string-templates": "2.0.34-alpha.5",
-    "@budibase/types": "2.0.34-alpha.5",
+    "@budibase/backend-core": "2.0.34-alpha.7",
+    "@budibase/client": "2.0.34-alpha.7",
+    "@budibase/pro": "2.0.34-alpha.6",
+    "@budibase/string-templates": "2.0.34-alpha.7",
+    "@budibase/types": "2.0.34-alpha.7",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -199,5 +199,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "cdfa70192e35b47e129974b03dc703ee4005c86d"
+  "gitHead": "899da8117bff87550e52d246e54c00bfe88bd56e"
 }

package/src/api/controllers/{webhook.js → webhook.ts}

@@ -1,66 +1,51 @@
-const { generateWebhookID, getWebhookParams } = require("../../db/utils")
+import { getWebhookParams } from "../../db/utils"
+import triggers from "../../automations/triggers"
+import { db as dbCore, context } from "@budibase/backend-core"
+import {
+  Webhook,
+  WebhookActionType,
+  BBContext,
+  Automation,
+} from "@budibase/types"
+import sdk from "../../sdk"
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
-const { WebhookType } = require("../../constants")
-const triggers = require("../../automations/triggers")
-const { getProdAppID } = require("@budibase/backend-core/db")
-const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
-function Webhook(name, type, target) {
-  this.live = true
-  this.name = name
-  this.action = {
-    type,
-    target,
-  }
-}
-
-exports.Webhook = Webhook
-
-exports.fetch = async ctx => {
-  const db = getAppDB()
+export async function fetch(ctx: BBContext) {
+  const db = context.getAppDB()
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
     })
   )
-  ctx.body = response.rows.map(row => row.doc)
+  ctx.body = response.rows.map((row: any) => row.doc)
 }
 
-exports.save = async ctx => {
-  const db = getAppDB()
-  const webhook = ctx.request.body
-  webhook.appId = ctx.appId
-
-  // check that the webhook exists
-  if (webhook._id) {
-    await db.get(webhook._id)
-  } else {
-    webhook._id = generateWebhookID()
-  }
-  const response = await db.put(webhook)
-  webhook._rev = response.rev
+export async function save(ctx: BBContext) {
+  const webhook = await sdk.automations.webhook.save(ctx.request.body)
   ctx.body = {
     message: "Webhook created successfully",
     webhook,
   }
 }
 
-exports.destroy = async ctx => {
-  const db = getAppDB()
-  ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
+export async function destroy(ctx: BBContext) {
+  ctx.body = await sdk.automations.webhook.destroy(
+    ctx.params.id,
+    ctx.params.rev
+  )
 }
 
-exports.buildSchema = async ctx => {
-  await updateAppId(ctx.params.instance)
-  const db = getAppDB()
-  const webhook = await db.get(ctx.params.id)
+export async function buildSchema(ctx: BBContext) {
+  await context.updateAppId(ctx.params.instance)
+  const db = context.getAppDB()
+  const webhook = (await db.get(ctx.params.id)) as Webhook
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
-  if (webhook.action.type === WebhookType.AUTOMATION) {
-    let automation = await db.get(webhook.action.target)
+  if (webhook.action.type === WebhookActionType.AUTOMATION) {
+    let automation = (await db.get(webhook.action.target)) as Automation
     const autoOutputs = automation.definition.trigger.schema.outputs
     let properties = webhook.bodySchema.properties
     // reset webhook outputs
@@ -78,18 +63,18 @@ exports.buildSchema = async ctx => {
   ctx.body = await db.put(webhook)
 }
 
-exports.trigger = async ctx => {
-  const prodAppId = getProdAppID(ctx.params.instance)
-  await updateAppId(prodAppId)
+export async function trigger(ctx: BBContext) {
+  const prodAppId = dbCore.getProdAppID(ctx.params.instance)
+  await context.updateAppId(prodAppId)
   try {
-    const db = getAppDB()
-    const webhook = await db.get(ctx.params.id)
+    const db = context.getAppDB()
+    const webhook = (await db.get(ctx.params.id)) as Webhook
     // validate against the schema
     if (webhook.bodySchema) {
       validate(ctx.request.body, webhook.bodySchema)
     }
     const target = await db.get(webhook.action.target)
-    if (webhook.action.type === WebhookType.AUTOMATION) {
+    if (webhook.action.type === WebhookActionType.AUTOMATION) {
       // trigger with both the pure request and then expand it
       // incase the user has produced a schema to bind to
       await triggers.externalTrigger(target, {
@@ -102,7 +87,7 @@ exports.trigger = async ctx => {
     ctx.body = {
       message: "Webhook trigger fired successfully",
     }
-  } catch (err) {
+  } catch (err: any) {
     if (err.status === 404) {
       ctx.status = 200
       ctx.body = {

package/src/api/routes/utils/validators.js

@@ -1,10 +1,10 @@
 const { joiValidator } = require("@budibase/backend-core/auth")
 const { DataSourceOperation } = require("../../../constants")
-const { WebhookType } = require("../../../constants")
 const {
   BUILTIN_PERMISSION_IDS,
   PermissionLevels,
 } = require("@budibase/backend-core/permissions")
+const { WebhookActionType } = require("@budibase/types")
 const Joi = require("joi")
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
@@ -126,7 +126,7 @@ exports.webhookValidator = () => {
     name: Joi.string().required(),
     bodySchema: Joi.object().optional(),
     action: Joi.object({
-      type: Joi.string().required().valid(WebhookType.AUTOMATION),
+      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
       target: Joi.string().required(),
     }).required(),
   }).unknown(true))

package/src/api/routes/{webhook.js → webhook.ts}

@@ -1,9 +1,10 @@
-const Router = require("@koa/router")
-const controller = require("../controllers/webhook")
-const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/backend-core/permissions")
-const { webhookValidator } = require("./utils/validators")
+import Router from "@koa/router"
+import * as controller from "../controllers/webhook"
+import authorized from "../../middleware/authorized"
+import { permissions } from "@budibase/backend-core"
+import { webhookValidator } from "./utils/validators"
 
+const BUILDER = permissions.BUILDER
 const router = new Router()
 
 router
@@ -23,4 +24,4 @@ router
   // this shouldn't have authorisation, right now its always public
   .post("/api/webhooks/trigger/:instance/:id", controller.trigger)
 
-module.exports = router
+export default router

package/src/app.ts

@@ -15,30 +15,16 @@ db.init()
 const Koa = require("koa")
 const destroyable = require("server-destroy")
 const koaBody = require("koa-body")
-const pino = require("koa-pino-logger")
 const http = require("http")
 const api = require("./api")
-const eventEmitter = require("./events")
 const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
-const fileSystem = require("./utilities/fileSystem")
-const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
-const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
-const fs = require("fs")
 import redis from "./utilities/redis"
-import * as migrations from "./migrations"
-import { events, installation, tenancy } from "@budibase/backend-core"
-import {
-  createAdminUser,
-  generateApiKey,
-  getChecklist,
-} from "./utilities/workerRequests"
-import { watch } from "./watch"
+import { events } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
-import sdk from "./sdk"
-import * as pro from "@budibase/pro"
+import { startup } from "./startup"
 
 const app = new Koa()
 
@@ -54,19 +40,6 @@ app.use(
   })
 )
 
-app.use(pino(pinoSettings()))
-
-if (!env.isTest()) {
-  const plugin = bullboard.init()
-  app.use(plugin)
-}
-
-app.context.eventEmitter = eventEmitter
-app.context.auth = {}
-
-// api routes
-app.use(api.router.routes())
-
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
   Sentry.init()
@@ -104,86 +77,8 @@ server.on("close", async () => {
   }
 })
 
-const initPro = async () => {
-  await pro.init({
-    backups: {
-      processing: {
-        exportAppFn: sdk.backups.exportApp,
-        importAppFn: sdk.backups.importApp,
-        statsFn: sdk.backups.calculateBackupStats,
-      },
-    },
-  })
-}
-
 module.exports = server.listen(env.PORT || 0, async () => {
-  console.log(`Budibase running on ${JSON.stringify(server.address())}`)
-  env._set("PORT", server.address().port)
-  eventEmitter.emitPort(env.PORT)
-  fileSystem.init()
-  await redis.init()
-
-  // run migrations on startup if not done via http
-  // not recommended in a clustered environment
-  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
-    try {
-      await migrations.migrate()
-    } catch (e) {
-      logAlert("Error performing migrations. Exiting.", e)
-      shutdown()
-    }
-  }
-
-  // check and create admin user if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.BB_ADMIN_USER_EMAIL &&
-    env.BB_ADMIN_USER_PASSWORD
-  ) {
-    const checklist = await getChecklist()
-    if (!checklist?.adminUser?.checked) {
-      try {
-        const tenantId = tenancy.getTenantId()
-        const user = await createAdminUser(
-          env.BB_ADMIN_USER_EMAIL,
-          env.BB_ADMIN_USER_PASSWORD,
-          tenantId
-        )
-        // Need to set up an API key for automated integration tests
-        if (env.isTest()) {
-          await generateApiKey(user._id)
-        }
-
-        console.log(
-          "Admin account automatically created for",
-          env.BB_ADMIN_USER_EMAIL
-        )
-      } catch (e) {
-        logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown()
-      }
-    }
-  }
-
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // done last - these will never complete
-  let promises = []
-  promises.push(automations.init())
-  promises.push(initPro())
-  await Promise.all(promises)
+  await startup(app, server)
 })
 
 const shutdown = () => {

package/src/automations/bullboard.js

@@ -3,6 +3,7 @@ const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
 const { queue } = require("@budibase/backend-core")
 const automation = require("../threads/automation")
+const { backups } = require("@budibase/pro")
 
 let automationQueue = queue.createQueue(
   queue.JobQueue.AUTOMATION,
@@ -11,9 +12,13 @@ let automationQueue = queue.createQueue(
 
 const PATH_PREFIX = "/bulladmin"
 
-exports.init = () => {
+exports.init = async () => {
   // Set up queues for bull board admin
+  const backupQueue = await backups.getBackupQueue()
   const queues = [automationQueue]
+  if (backupQueue) {
+    queues.push(backupQueue)
+  }
   const adapters = []
   const serverAdapter = new KoaAdapter()
   for (let queue of queues) {

package/src/automations/utils.ts

@@ -1,10 +1,9 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
-import * as webhooks from "../api/controllers/webhook"
 import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
-import { MetadataTypes, WebhookType } from "../constants"
+import { MetadataTypes } from "../constants"
 import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
@@ -15,7 +14,8 @@ import {
 } from "@budibase/backend-core/context"
 import { context } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
-import { Automation } from "@budibase/types"
+import { Automation, WebhookActionType } from "@budibase/types"
+import sdk from "../sdk"
 
 const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
@@ -197,16 +197,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
       let db = getAppDB()
       // need to get the webhook to get the rev
       const webhook = await db.get(oldTrigger.webhookId)
-      const ctx = {
-        appId,
-        params: { id: webhook._id, rev: webhook._rev },
-      }
       // might be updating - reset the inputs to remove the URLs
       if (newTrigger) {
         delete newTrigger.webhookId
         newTrigger.inputs = {}
       }
-      await webhooks.destroy(ctx)
+      await sdk.automations.webhook.destroy(webhook._id, webhook._rev)
     } catch (err) {
       // don't worry about not being able to delete, if it doesn't exist all good
     }
@@ -216,18 +212,14 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx: any = {
-      appId,
-      request: {
-        body: new webhooks.Webhook(
-          "Automation webhook",
-          WebhookType.AUTOMATION,
-          newAuto._id
-        ),
-      },
-    }
-    await webhooks.save(ctx)
-    const id = ctx.body.webhook._id
+    const webhook = await sdk.automations.webhook.save(
+      sdk.automations.webhook.newDoc(
+        "Automation webhook",
+        WebhookActionType.AUTOMATION,
+        newAuto._id
+      )
+    )
+    const id = webhook._id
     newTrigger.webhookId = id
     // the app ID has to be development for this endpoint
     // it can only be used when building the app

package/src/constants/index.js

@@ -196,10 +196,6 @@ exports.BuildSchemaErrors = {
   INVALID_COLUMN: "invalid_column",
 }
 
-exports.WebhookType = {
-  AUTOMATION: "automation",
-}
-
 exports.AutomationErrors = {
   INCORRECT_TYPE: "INCORRECT_TYPE",
   MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",

package/src/definitions/automations.ts

@@ -1,9 +1,4 @@
-import {
-  Automation,
-  AutomationResults,
-  AutomationStep,
-  Document,
-} from "@budibase/types"
+import { AutomationResults, AutomationStep, Document } from "@budibase/types"
 
 export enum LoopStepType {
   ARRAY = "Array",

package/src/migrations/functions/tests/syncQuotas.spec.js

@@ -1,13 +1,11 @@
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
 const syncApps = jest.fn()
 const syncRows = jest.fn()
 const syncPlugins = jest.fn()
-
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
 jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const migration = require("../syncQuotas")
 
 describe("run", () => {

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -1,11 +1,13 @@
+jest.mock("@budibase/backend-core/db", () => ({
+  ...jest.requireActual("@budibase/backend-core/db"),
+  createNewUserEmailView: jest.fn(),
+}))
+const coreDb = require("@budibase/backend-core/db")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { TENANT_ID } = require("../../../tests/utilities/structures")
 const { getGlobalDB, doInTenant } = require("@budibase/backend-core/tenancy")
 
 // mock email view creation
-const coreDb = require("@budibase/backend-core/db")
-const createNewUserEmailView = jest.fn()
-coreDb.createNewUserEmailView = createNewUserEmailView
 
 const migration = require("../userEmailViewCasing")
 
@@ -22,7 +24,7 @@ describe("run", () => {
       await doInTenant(TENANT_ID, async () => {
         const globalDb = getGlobalDB()
         await migration.run(globalDb)
-        expect(createNewUserEmailView).toHaveBeenCalledTimes(1)
+        expect(coreDb.createNewUserEmailView).toHaveBeenCalledTimes(1)
       })
     })
 })

package/src/sdk/app/automations/index.ts

@@ -0,0 +1,5 @@
+import * as webhook from "./webhook"
+
+export default {
+  webhook,
+}

package/src/sdk/app/automations/webhook.ts

@@ -0,0 +1,43 @@
+import { Webhook, WebhookActionType } from "@budibase/types"
+import { db as dbCore, context } from "@budibase/backend-core"
+import { generateWebhookID } from "../../../db/utils"
+
+function isWebhookID(id: string) {
+  return id.startsWith(dbCore.DocumentType.WEBHOOK)
+}
+
+export function newDoc(
+  name: string,
+  type: WebhookActionType,
+  target: string
+): Webhook {
+  return {
+    live: true,
+    name,
+    action: {
+      type,
+      target,
+    },
+  }
+}
+
+export async function save(webhook: Webhook) {
+  const db = context.getAppDB()
+  // check that the webhook exists
+  if (webhook._id && isWebhookID(webhook._id)) {
+    await db.get(webhook._id)
+  } else {
+    webhook._id = generateWebhookID()
+  }
+  const response = await db.put(webhook)
+  webhook._rev = response.rev
+  return webhook
+}
+
+export async function destroy(id: string, rev: string) {
+  const db = context.getAppDB()
+  if (!id || !isWebhookID(id)) {
+    throw new Error("Provided webhook ID is not valid.")
+  }
+  return await db.remove(id, rev)
+}

package/src/sdk/app/backups/imports.ts

@@ -1,20 +1,25 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import {
-  uploadDirectory,
   upload,
+  uploadDirectory,
 } from "../../../utilities/fileSystem/utilities"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
+import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-import { CouchFindOptions, RowAttachment } from "@budibase/types"
+import {
+  Automation,
+  AutomationTriggerStepId,
+  CouchFindOptions,
+  RowAttachment,
+} from "@budibase/types"
+import PouchDB from "pouchdb"
 const uuid = require("uuid/v4")
 const tar = require("tar")
-import PouchDB from "pouchdb"
 
 type TemplateType = {
   file?: {
@@ -81,6 +86,34 @@ async function updateAttachmentColumns(
   }
 }
 
+async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
+  const automations = (
+    await db.allDocs(
+      getAutomationParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc) as Automation[]
+  const devAppId = dbCore.getDevAppID(prodAppId)
+  let toSave: Automation[] = []
+  for (let automation of automations) {
+    const oldDevAppId = automation.appId,
+      oldProdAppId = dbCore.getProdAppID(automation.appId)
+    if (
+      automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK
+    ) {
+      const old = automation.definition.trigger.inputs
+      automation.definition.trigger.inputs = {
+        schemaUrl: old.schemaUrl.replace(oldDevAppId, devAppId),
+        triggerUrl: old.triggerUrl.replace(oldProdAppId, prodAppId),
+      }
+    }
+    automation.appId = devAppId
+    toSave.push(automation)
+  }
+  await db.bulkDocs(toSave)
+}
+
 /**
  * This function manages temporary template files which are stored by Koa.
  * @param {Object} template The template object retrieved from the Koa context object.
@@ -165,5 +198,6 @@ export async function importApp(
     throw "Error loading database dump from template."
   }
   await updateAttachmentColumns(prodAppId, db)
+  await updateAutomations(prodAppId, db)
   return ok
 }

package/src/sdk/index.ts

@@ -1,9 +1,11 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
+import { default as automations } from "./app/automations"
 
 const sdk = {
   backups,
   tables,
+  automations,
 }
 
 // default export for TS