@budibase/server 1.0.199 → 1.0.200-alpha.2

package/src/migrations/index.ts

@@ -1,7 +1,6 @@
-const {
-  MIGRATION_TYPES,
-  runMigrations,
-} = require("@budibase/backend-core/migrations")
+import { migrations, redis } from "@budibase/backend-core"
+import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
+import env from "../environment"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
@@ -9,59 +8,129 @@ import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
 import * as developerQuota from "./functions/developerQuota"
 import * as publishedAppsQuota from "./functions/publishedAppsQuota"
-
-export interface Migration {
-  type: string
-  name: string
-  opts?: object
-  fn: Function
-}
+import * as backfill from "./functions/backfill"
 
 /**
- * e.g.
- * {
- *   tenantIds: ['bb'],
- *   force: {
- *    global: ['quota_1']
- *   }
- * }
+ * Populate the migration function and additional configuration from
+ * the static migration definitions.
  */
-export interface MigrationOptions {
-  tenantIds?: string[]
-  force?: {
-    [type: string]: string[]
+export const buildMigrations = () => {
+  const definitions = migrations.DEFINITIONS
+  const serverMigrations: Migration[] = []
+
+  for (const definition of definitions) {
+    switch (definition.name) {
+      case MigrationName.USER_EMAIL_VIEW_CASING: {
+        serverMigrations.push({
+          ...definition,
+          fn: userEmailViewCasing.run,
+        })
+        break
+      }
+      case MigrationName.QUOTAS_1: {
+        serverMigrations.push({
+          ...definition,
+          fn: quota1.run,
+        })
+        break
+      }
+      case MigrationName.APP_URLS: {
+        serverMigrations.push({
+          ...definition,
+          appOpts: { all: true },
+          fn: appUrls.run,
+        })
+        break
+      }
+      case MigrationName.DEVELOPER_QUOTA: {
+        serverMigrations.push({
+          ...definition,
+          fn: developerQuota.run,
+        })
+        break
+      }
+      case MigrationName.PUBLISHED_APP_QUOTA: {
+        serverMigrations.push({
+          ...definition,
+          fn: publishedAppsQuota.run,
+        })
+        break
+      }
+      case MigrationName.EVENT_APP_BACKFILL: {
+        serverMigrations.push({
+          ...definition,
+          appOpts: { all: true },
+          fn: backfill.app.run,
+          silent: !!env.SELF_HOSTED, // reduce noisy logging
+          preventRetry: !!env.SELF_HOSTED, // only ever run once
+        })
+        break
+      }
+      case MigrationName.EVENT_GLOBAL_BACKFILL: {
+        serverMigrations.push({
+          ...definition,
+          fn: backfill.global.run,
+          silent: !!env.SELF_HOSTED, // reduce noisy logging
+          preventRetry: !!env.SELF_HOSTED, // only ever run once
+        })
+        break
+      }
+      case MigrationName.EVENT_INSTALLATION_BACKFILL: {
+        serverMigrations.push({
+          ...definition,
+          fn: backfill.installation.run,
+          silent: !!env.SELF_HOSTED, // reduce noisy logging
+          preventRetry: !!env.SELF_HOSTED, // only ever run once
+        })
+        break
+      }
+    }
   }
+
+  return serverMigrations
 }
 
-export const MIGRATIONS: Migration[] = [
-  {
-    type: MIGRATION_TYPES.GLOBAL,
-    name: "user_email_view_casing",
-    fn: userEmailViewCasing.run,
-  },
-  {
-    type: MIGRATION_TYPES.GLOBAL,
-    name: "quotas_1",
-    fn: quota1.run,
-  },
-  {
-    type: MIGRATION_TYPES.APP,
-    name: "app_urls",
-    opts: { all: true },
-    fn: appUrls.run,
-  },
-  {
-    type: MIGRATION_TYPES.GLOBAL,
-    name: "developer_quota",
-    fn: developerQuota.run,
-  },
-  {
-    type: MIGRATION_TYPES.GLOBAL,
-    name: "published_apps_quota",
-    fn: publishedAppsQuota.run,
-  },
-]
+export const MIGRATIONS = buildMigrations()
 
 export const migrate = async (options?: MigrationOptions) => {
-  await runMigrations(MIGRATIONS, options)
+  if (env.SELF_HOSTED) {
+    // self host runs migrations on startup
+    // make sure only a single instance runs them
+    await migrateWithLock(options)
+  } else {
+    await migrations.runMigrations(MIGRATIONS, options)
+  }
+}
+
+const migrateWithLock = async (options?: MigrationOptions) => {
+  // get a new lock client
+
+  const redlock = await redis.clients.getMigrationsRedlock()
+  // lock for 15 minutes
+  const ttl = 1000 * 60 * 15
+
+  let migrationLock
+
+  // acquire lock
+  try {
+    migrationLock = await redlock.lock("migrations", ttl)
+  } catch (e: any) {
+    if (e.name === "LockError") {
+      return
+    } else {
+      throw e
+    }
+  }
+
+  // run migrations
+  try {
+    await migrations.runMigrations(MIGRATIONS, options)
+  } finally {
+    // release lock
+    try {
+      await migrationLock.unlock()
+    } catch (e) {
+      console.error("unable to release migration lock")
+    }
+  }
 }

package/src/migrations/tests/helpers.ts

@@ -0,0 +1,40 @@
+// Mimic configs test configuration from worker, creation configs directly in database
+
+import * as structures from "./structures"
+import { db } from "@budibase/backend-core"
+import { Config } from "@budibase/types"
+
+export const saveSettingsConfig = async (globalDb: any) => {
+  const config = structures.settings()
+  await saveConfig(config, globalDb)
+}
+
+export const saveGoogleConfig = async (globalDb: any) => {
+  const config = structures.google()
+  await saveConfig(config, globalDb)
+}
+
+export const saveOIDCConfig = async (globalDb: any) => {
+  const config = structures.oidc()
+  await saveConfig(config, globalDb)
+}
+
+export const saveSmtpConfig = async (globalDb: any) => {
+  const config = structures.smtp()
+  await saveConfig(config, globalDb)
+}
+
+const saveConfig = async (config: Config, globalDb: any) => {
+  config._id = db.generateConfigID({ type: config.type })
+
+  let response
+  try {
+    response = await globalDb.get(config._id)
+    config._rev = response._rev
+    await globalDb.put(config)
+  } catch (e: any) {
+    if (e.status === 404) {
+      await globalDb.put(config)
+    }
+  }
+}

package/src/migrations/tests/index.spec.ts

@@ -0,0 +1,132 @@
+import {
+  events,
+  migrations,
+  tenancy,
+  DocumentTypes,
+  context,
+  db,
+} from "@budibase/backend-core"
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import structures from "../../tests/utilities/structures"
+import { MIGRATIONS } from "../"
+import * as helpers from "./helpers"
+
+const { mocks } = require("@budibase/backend-core/tests")
+const timestamp = mocks.date.MOCK_DATE.toISOString()
+
+const clearMigrations = async () => {
+  const dbs = [context.getDevAppDB(), context.getProdAppDB()]
+  for (const db of dbs) {
+    const doc = await db.get(DocumentTypes.MIGRATIONS)
+    const newDoc = { _id: doc._id, _rev: doc._rev }
+    await db.put(newDoc)
+  }
+}
+
+describe("migrations", () => {
+  const config = new TestConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll(() => {
+    config.end()
+  })
+
+  describe("backfill", () => {
+    it("runs app db migration", async () => {
+      await config.doInContext(null, async () => {
+        await clearMigrations()
+        await config.createAutomation()
+        await config.createAutomation(structures.newAutomation())
+        await config.createDatasource()
+        await config.createDatasource()
+        await config.createLayout()
+        await config.createQuery()
+        await config.createQuery()
+        await config.createRole()
+        await config.createRole()
+        await config.createTable()
+        await config.createView()
+        await config.createTable()
+        await config.createView(structures.view(config.table._id))
+        await config.createScreen()
+        await config.createScreen()
+
+        jest.clearAllMocks()
+        const migration = MIGRATIONS.filter(
+          m => m.name === "event_app_backfill"
+        )[0]
+        await migrations.runMigration(migration)
+
+        expect(events.app.created).toBeCalledTimes(1)
+        expect(events.app.published).toBeCalledTimes(1)
+        expect(events.automation.created).toBeCalledTimes(2)
+        expect(events.automation.stepCreated).toBeCalledTimes(1)
+        expect(events.datasource.created).toBeCalledTimes(2)
+        expect(events.layout.created).toBeCalledTimes(1)
+        expect(events.query.created).toBeCalledTimes(2)
+        expect(events.role.created).toBeCalledTimes(2)
+        expect(events.table.created).toBeCalledTimes(3)
+        expect(events.view.created).toBeCalledTimes(2)
+        expect(events.view.calculationCreated).toBeCalledTimes(1)
+        expect(events.view.filterCreated).toBeCalledTimes(1)
+        expect(events.screen.created).toBeCalledTimes(2)
+        expect(events.backfill.appSucceeded).toBeCalledTimes(2)
+
+        const processor = events.processors.analyticsProcessor.processEvent
+        console.log(processor)
+
+        // to make sure caching is working as expected
+        expect(
+          events.processors.analyticsProcessor.processEvent
+        ).toBeCalledTimes(23)
+      })
+    })
+  })
+
+  it("runs global db migration", async () => {
+    await config.doInContext(null, async () => {
+      await clearMigrations()
+      const appId = config.prodAppId
+      const roles = { [appId]: "role_12345" }
+      await config.createUser(undefined, undefined, false, true, roles) // admin only
+      await config.createUser(undefined, undefined, false, false, roles) // non admin non builder
+      await config.createTable()
+      await config.createRow()
+      await config.createRow()
+
+      const db = tenancy.getGlobalDB()
+      await helpers.saveGoogleConfig(db)
+      await helpers.saveOIDCConfig(db)
+      await helpers.saveSettingsConfig(db)
+      await helpers.saveSmtpConfig(db)
+
+      jest.clearAllMocks()
+      const migration = MIGRATIONS.filter(
+        m => m.name === "event_global_backfill"
+      )[0]
+      await migrations.runMigration(migration)
+
+      expect(events.user.created).toBeCalledTimes(3)
+      expect(events.role.assigned).toBeCalledTimes(2)
+      expect(events.user.permissionBuilderAssigned).toBeCalledTimes(1) // default test user
+      expect(events.user.permissionAdminAssigned).toBeCalledTimes(1) // admin from above
+      expect(events.rows.created).toBeCalledTimes(1)
+      expect(events.rows.created).toBeCalledWith(2, timestamp)
+      expect(events.email.SMTPCreated).toBeCalledTimes(1)
+      expect(events.auth.SSOCreated).toBeCalledTimes(2)
+      expect(events.auth.SSOActivated).toBeCalledTimes(2)
+      expect(events.org.logoUpdated).toBeCalledTimes(1)
+      expect(events.org.nameUpdated).toBeCalledTimes(1)
+      expect(events.org.platformURLUpdated).toBeCalledTimes(1)
+      expect(events.backfill.tenantSucceeded).toBeCalledTimes(1)
+
+      // to make sure caching is working as expected
+      expect(events.processors.analyticsProcessor.processEvent).toBeCalledTimes(
+        19
+      )
+    })
+  })
+})

package/src/migrations/tests/structures.ts

@@ -0,0 +1,66 @@
+import { utils } from "@budibase/backend-core"
+import {
+  SMTPConfig,
+  OIDCConfig,
+  GoogleConfig,
+  SettingsConfig,
+  ConfigType,
+} from "@budibase/types"
+
+export const oidc = (conf?: OIDCConfig): OIDCConfig => {
+  return {
+    type: ConfigType.OIDC,
+    config: {
+      configs: [
+        {
+          configUrl: "http://someconfigurl",
+          clientID: "clientId",
+          clientSecret: "clientSecret",
+          logo: "Microsoft",
+          name: "Active Directory",
+          uuid: utils.newid(),
+          activated: true,
+          ...conf,
+        },
+      ],
+    },
+  }
+}
+
+export const google = (conf?: GoogleConfig): GoogleConfig => {
+  return {
+    type: ConfigType.GOOGLE,
+    config: {
+      clientID: "clientId",
+      clientSecret: "clientSecret",
+      activated: true,
+      ...conf,
+    },
+  }
+}
+
+export const smtp = (conf?: SMTPConfig): SMTPConfig => {
+  return {
+    type: ConfigType.SMTP,
+    config: {
+      port: 12345,
+      host: "smtptesthost.com",
+      from: "testfrom@test.com",
+      subject: "Hello!",
+      secure: false,
+      ...conf,
+    },
+  }
+}
+
+export const settings = (conf?: SettingsConfig): SettingsConfig => {
+  return {
+    type: ConfigType.SETTINGS,
+    config: {
+      platformUrl: "http://mycustomdomain.com",
+      logoUrl: "http://mylogourl,com",
+      company: "mycompany",
+      ...conf,
+    },
+  }
+}

package/src/module.d.ts

@@ -1,4 +1,3 @@
-declare module "@budibase/backend-core"
 declare module "@budibase/backend-core/tenancy"
 declare module "@budibase/backend-core/db"
 declare module "@budibase/backend-core/context"