@budibase/server 2.0.34-alpha.1 → 2.0.34-alpha.10

package/src/migrations/functions/tests/syncQuotas.spec.js

@@ -1,13 +1,11 @@
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
 const syncApps = jest.fn()
 const syncRows = jest.fn()
 const syncPlugins = jest.fn()
-
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
 jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const migration = require("../syncQuotas")
 
 describe("run", () => {

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -1,11 +1,13 @@
+jest.mock("@budibase/backend-core/db", () => ({
+  ...jest.requireActual("@budibase/backend-core/db"),
+  createNewUserEmailView: jest.fn(),
+}))
+const coreDb = require("@budibase/backend-core/db")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { TENANT_ID } = require("../../../tests/utilities/structures")
 const { getGlobalDB, doInTenant } = require("@budibase/backend-core/tenancy")
 
 // mock email view creation
-const coreDb = require("@budibase/backend-core/db")
-const createNewUserEmailView = jest.fn()
-coreDb.createNewUserEmailView = createNewUserEmailView
 
 const migration = require("../userEmailViewCasing")
 
@@ -22,7 +24,7 @@ describe("run", () => {
       await doInTenant(TENANT_ID, async () => {
         const globalDb = getGlobalDB()
         await migration.run(globalDb)
-        expect(createNewUserEmailView).toHaveBeenCalledTimes(1)
+        expect(coreDb.createNewUserEmailView).toHaveBeenCalledTimes(1)
       })
     })
 })

package/src/sdk/app/automations/index.ts

@@ -0,0 +1,5 @@
+import * as webhook from "./webhook"
+
+export default {
+  webhook,
+}

package/src/sdk/app/automations/webhook.ts

@@ -0,0 +1,43 @@
+import { Webhook, WebhookActionType } from "@budibase/types"
+import { db as dbCore, context } from "@budibase/backend-core"
+import { generateWebhookID } from "../../../db/utils"
+
+function isWebhookID(id: string) {
+  return id.startsWith(dbCore.DocumentType.WEBHOOK)
+}
+
+export function newDoc(
+  name: string,
+  type: WebhookActionType,
+  target: string
+): Webhook {
+  return {
+    live: true,
+    name,
+    action: {
+      type,
+      target,
+    },
+  }
+}
+
+export async function save(webhook: Webhook) {
+  const db = context.getAppDB()
+  // check that the webhook exists
+  if (webhook._id && isWebhookID(webhook._id)) {
+    await db.get(webhook._id)
+  } else {
+    webhook._id = generateWebhookID()
+  }
+  const response = await db.put(webhook)
+  webhook._rev = response.rev
+  return webhook
+}
+
+export async function destroy(id: string, rev: string) {
+  const db = context.getAppDB()
+  if (!id || !isWebhookID(id)) {
+    throw new Error("Provided webhook ID is not valid.")
+  }
+  return await db.remove(id, rev)
+}

package/src/sdk/app/backups/imports.ts

@@ -1,20 +1,25 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import {
-  uploadDirectory,
   upload,
+  uploadDirectory,
 } from "../../../utilities/fileSystem/utilities"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
+import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
-import { CouchFindOptions, RowAttachment } from "@budibase/types"
+import {
+  Automation,
+  AutomationTriggerStepId,
+  CouchFindOptions,
+  RowAttachment,
+} from "@budibase/types"
+import PouchDB from "pouchdb"
 const uuid = require("uuid/v4")
 const tar = require("tar")
-import PouchDB from "pouchdb"
 
 type TemplateType = {
   file?: {
@@ -81,12 +86,43 @@ async function updateAttachmentColumns(
   }
 }
 
+async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
+  const automations = (
+    await db.allDocs(
+      getAutomationParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc) as Automation[]
+  const devAppId = dbCore.getDevAppID(prodAppId)
+  let toSave: Automation[] = []
+  for (let automation of automations) {
+    const oldDevAppId = automation.appId,
+      oldProdAppId = dbCore.getProdAppID(automation.appId)
+    if (
+      automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK
+    ) {
+      const old = automation.definition.trigger.inputs
+      automation.definition.trigger.inputs = {
+        schemaUrl: old.schemaUrl.replace(oldDevAppId, devAppId),
+        triggerUrl: old.triggerUrl.replace(oldProdAppId, prodAppId),
+      }
+    }
+    automation.appId = devAppId
+    toSave.push(automation)
+  }
+  await db.bulkDocs(toSave)
+}
+
 /**
  * This function manages temporary template files which are stored by Koa.
  * @param {Object} template The template object retrieved from the Koa context object.
  * @returns {Object} Returns a fs read stream which can be loaded into the database.
  */
 async function getTemplateStream(template: TemplateType) {
+  if (template.file && template.file.type !== "text/plain") {
+    throw new Error("Cannot import a non-text based file.")
+  }
   if (template.file) {
     return fs.createReadStream(template.file.path)
   } else if (template.key) {
@@ -123,7 +159,7 @@ export async function importApp(
 ) {
   let prodAppId = dbCore.getProdAppID(appId)
   let dbStream: any
-  const isTar = template.file && template.file.type === "application/gzip"
+  const isTar = template.file && template?.file?.type?.endsWith("gzip")
   const isDirectory =
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
@@ -165,5 +201,6 @@ export async function importApp(
     throw "Error loading database dump from template."
   }
   await updateAttachmentColumns(prodAppId, db)
+  await updateAutomations(prodAppId, db)
   return ok
 }

package/src/sdk/index.ts

@@ -1,9 +1,11 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
+import { default as automations } from "./app/automations"
 
 const sdk = {
   backups,
   tables,
+  automations,
 }
 
 // default export for TS

package/src/startup.ts

@@ -0,0 +1,138 @@
+import * as env from "./environment"
+import redis from "./utilities/redis"
+import {
+  createAdminUser,
+  generateApiKey,
+  getChecklist,
+} from "./utilities/workerRequests"
+import {
+  installation,
+  pinoSettings,
+  tenancy,
+  logging,
+} from "@budibase/backend-core"
+import fs from "fs"
+import { watch } from "./watch"
+import automations from "./automations"
+import fileSystem from "./utilities/fileSystem"
+import eventEmitter from "./events"
+import * as migrations from "./migrations"
+import bullboard from "./automations/bullboard"
+import * as pro from "@budibase/pro"
+import api from "./api"
+import sdk from "./sdk"
+const pino = require("koa-pino-logger")
+
+let STARTUP_RAN = false
+
+async function initRoutes(app: any) {
+  app.use(pino(pinoSettings()))
+
+  if (!env.isTest()) {
+    const plugin = await bullboard.init()
+    app.use(plugin)
+  }
+
+  app.context.eventEmitter = eventEmitter
+  app.context.auth = {}
+
+  // api routes
+  app.use(api.router.routes())
+}
+
+async function initPro() {
+  await pro.init({
+    backups: {
+      processing: {
+        exportAppFn: sdk.backups.exportApp,
+        importAppFn: sdk.backups.importApp,
+        statsFn: sdk.backups.calculateBackupStats,
+      },
+    },
+  })
+}
+
+function shutdown(server?: any) {
+  server.close()
+  server.destroy()
+}
+
+export async function startup(app?: any, server?: any) {
+  if (STARTUP_RAN) {
+    return
+  }
+  STARTUP_RAN = true
+  if (server) {
+    console.log(`Budibase running on ${JSON.stringify(server.address())}`)
+    env._set("PORT", server.address().port)
+  }
+  eventEmitter.emitPort(env.PORT)
+  fileSystem.init()
+  await redis.init()
+
+  // run migrations on startup if not done via http
+  // not recommended in a clustered environment
+  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
+    try {
+      await migrations.migrate()
+    } catch (e) {
+      logging.logAlert("Error performing migrations. Exiting.", e)
+      shutdown()
+    }
+  }
+
+  // check and create admin user if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.BB_ADMIN_USER_EMAIL &&
+    env.BB_ADMIN_USER_PASSWORD
+  ) {
+    const checklist = await getChecklist()
+    if (!checklist?.adminUser?.checked) {
+      try {
+        const tenantId = tenancy.getTenantId()
+        const user = await createAdminUser(
+          env.BB_ADMIN_USER_EMAIL,
+          env.BB_ADMIN_USER_PASSWORD,
+          tenantId
+        )
+        // Need to set up an API key for automated integration tests
+        if (env.isTest()) {
+          await generateApiKey(user._id)
+        }
+
+        console.log(
+          "Admin account automatically created for",
+          env.BB_ADMIN_USER_EMAIL
+        )
+      } catch (e) {
+        logging.logAlert("Error creating initial admin user. Exiting.", e)
+        shutdown()
+      }
+    }
+  }
+
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
+}

package/src/tests/utilities/TestConfiguration.js

@@ -26,6 +26,7 @@ const context = require("@budibase/backend-core/context")
 const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
 const { encrypt } = require("@budibase/backend-core/encryption")
 const { DocumentType, generateUserMetadataID } = require("../../db/utils")
+const { startup } = require("../../startup")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -41,6 +42,9 @@ class TestConfiguration {
       this.server = require("../../app")
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
+      this.started = true
+    } else {
+      this.started = false
     }
     this.appId = null
     this.allApps = []
@@ -95,6 +99,9 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
+    if (!this.started) {
+      await startup()
+    }
     this.user = await this.globalUser()
     this.globalUserId = this.user._id
     this.userMetadataId = generateUserMetadataID(this.globalUserId)