@budibase/server 1.0.198 → 1.0.199-alpha.0

package/src/migrations/functions/backfill/app.ts

@@ -0,0 +1,148 @@
+import * as automations from "./app/automations"
+import * as datasources from "./app/datasources"
+import * as layouts from "./app/layouts"
+import * as queries from "./app/queries"
+import * as roles from "./app/roles"
+import * as tables from "./app/tables"
+import * as screens from "./app/screens"
+import * as global from "./global"
+import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
+import { db as dbUtils, events } from "@budibase/backend-core"
+import env from "../../../environment"
+import { DEFAULT_TIMESTAMP } from "."
+
+const failGraceful = env.SELF_HOSTED && !env.isDev()
+
+const handleError = (e: any, errors?: any) => {
+  if (failGraceful) {
+    if (errors) {
+      errors.push(e)
+    }
+    return
+  }
+  throw e
+}
+
+const EVENTS = [
+  Event.AUTOMATION_CREATED,
+  Event.AUTOMATION_STEP_CREATED,
+  Event.DATASOURCE_CREATED,
+  Event.LAYOUT_CREATED,
+  Event.QUERY_CREATED,
+  Event.ROLE_CREATED,
+  Event.SCREEN_CREATED,
+  Event.TABLE_CREATED,
+  Event.VIEW_CREATED,
+  Event.VIEW_CALCULATION_CREATED,
+  Event.VIEW_FILTER_CREATED,
+  Event.APP_PUBLISHED,
+  Event.APP_CREATED,
+]
+
+/**
+ * Date:
+ * May 2022
+ *
+ * Description:
+ * Backfill app events.
+ */
+
+export const run = async (appDb: any) => {
+  try {
+    if (await global.isComplete()) {
+      // make sure new apps aren't backfilled
+      // return if the global migration for this tenant is complete
+      // which runs after the app migrations
+      return
+    }
+
+    // tell the event pipeline to start caching
+    // events for this tenant
+    await events.backfillCache.start(EVENTS)
+
+    let timestamp: string | number = DEFAULT_TIMESTAMP
+    const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA)
+    if (app.createdAt) {
+      timestamp = app.createdAt as string
+    }
+
+    if (dbUtils.isProdAppID(app.appId)) {
+      await events.app.published(app, timestamp)
+    }
+
+    const totals: any = {}
+    const errors: any = []
+
+    if (dbUtils.isDevAppID(app.appId)) {
+      await events.app.created(app, timestamp)
+      try {
+        totals.automations = await automations.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.datasources = await datasources.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.layouts = await layouts.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.queries = await queries.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.roles = await roles.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.screens = await screens.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.tables = await tables.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+    }
+
+    const properties: AppBackfillSucceededEvent = {
+      appId: app.appId,
+      automations: totals.automations,
+      datasources: totals.datasources,
+      layouts: totals.layouts,
+      queries: totals.queries,
+      roles: totals.roles,
+      tables: totals.tables,
+      screens: totals.screens,
+    }
+
+    if (errors.length) {
+      properties.errors = errors.map((e: any) =>
+        JSON.stringify(e, Object.getOwnPropertyNames(e))
+      )
+      properties.errorCount = errors.length
+    } else {
+      properties.errorCount = 0
+    }
+
+    await events.backfill.appSucceeded(properties)
+    // tell the event pipeline to stop caching events for this tenant
+    await events.backfillCache.end()
+  } catch (e) {
+    handleError(e)
+    await events.backfill.appFailed(e)
+  }
+}

package/src/migrations/functions/backfill/global/configs.ts

@@ -0,0 +1,66 @@
+import { events, db as dbUtils } from "@budibase/backend-core"
+import {
+  Config,
+  isSMTPConfig,
+  isGoogleConfig,
+  isOIDCConfig,
+  isSettingsConfig,
+} from "@budibase/types"
+import env from "./../../../../environment"
+
+const getConfigs = async (globalDb: any): Promise<Config[]> => {
+  const response = await globalDb.allDocs(
+    dbUtils.getConfigParams(
+      {},
+      {
+        include_docs: true,
+      }
+    )
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (
+  globalDb: any,
+  timestamp: string | number | undefined
+) => {
+  const configs = await getConfigs(globalDb)
+
+  for (const config of configs) {
+    if (isSMTPConfig(config)) {
+      await events.email.SMTPCreated(timestamp)
+    }
+    if (isGoogleConfig(config)) {
+      await events.auth.SSOCreated("google", timestamp)
+      if (config.config.activated) {
+        await events.auth.SSOActivated("google", timestamp)
+      }
+    }
+    if (isOIDCConfig(config)) {
+      await events.auth.SSOCreated("oidc", timestamp)
+      if (config.config.configs[0].activated) {
+        await events.auth.SSOActivated("oidc", timestamp)
+      }
+    }
+    if (isSettingsConfig(config)) {
+      const company = config.config.company
+      if (company && company !== "Budibase") {
+        await events.org.nameUpdated(timestamp)
+      }
+
+      const logoUrl = config.config.logoUrl
+      if (logoUrl) {
+        await events.org.logoUpdated(timestamp)
+      }
+
+      const platformUrl = config.config.platformUrl
+      if (
+        platformUrl &&
+        platformUrl !== "http://localhost:10000" &&
+        env.SELF_HOSTED
+      ) {
+        await events.org.platformURLUpdated(timestamp)
+      }
+    }
+  }
+}

package/src/migrations/functions/backfill/global/quotas.ts

@@ -0,0 +1,60 @@
+import { DEFAULT_TIMESTAMP } from "./../index"
+import { events } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { App } from "@budibase/types"
+
+const getOldestCreatedAt = (allApps: App[]): string | undefined => {
+  const timestamps = allApps
+    .filter(app => !!app.createdAt)
+    .map(app => app.createdAt as string)
+    .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
+
+  if (timestamps.length) {
+    return timestamps[0]
+  }
+}
+
+const getMonthTimestamp = (monthString: string): number => {
+  const parts = monthString.split("-")
+  const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
+  const year = parseInt(parts[1])
+
+  // using 0 as the day in next month gives us last day in previous month
+  const date = new Date(year, month + 1, 0).getTime()
+  const now = new Date().getTime()
+
+  if (date > now) {
+    return now
+  } else {
+    return date
+  }
+}
+
+export const backfill = async (allApps: App[]) => {
+  const usage = await quotas.getQuotaUsage()
+
+  const rows = usage.usageQuota.rows
+  let timestamp: string | number = DEFAULT_TIMESTAMP
+
+  const oldestAppTimestamp = getOldestCreatedAt(allApps)
+  if (oldestAppTimestamp) {
+    timestamp = oldestAppTimestamp
+  }
+
+  await events.rows.created(rows, timestamp)
+
+  for (const [monthString, quotas] of Object.entries(usage.monthly)) {
+    if (monthString === "current") {
+      continue
+    }
+    const monthTimestamp = getMonthTimestamp(monthString)
+
+    const queries = quotas.queries
+    await events.query.run(queries, monthTimestamp)
+
+    const automations = quotas.automations
+    await events.automation.run(automations, monthTimestamp)
+  }
+
+  return usage
+}

package/src/migrations/functions/backfill/global/users.ts

@@ -0,0 +1,49 @@
+import { events, db as dbUtils } from "@budibase/backend-core"
+import { User, CloudAccount } from "@budibase/types"
+import { DEFAULT_TIMESTAMP } from ".."
+
+// manually define user doc params - normally server doesn't read users from the db
+const getUserParams = (props: any) => {
+  return dbUtils.getDocParams(dbUtils.DocumentTypes.USER, null, props)
+}
+
+export const getUsers = async (globalDb: any): Promise<User[]> => {
+  const response = await globalDb.allDocs(
+    getUserParams({
+      include_docs: true,
+    })
+  )
+  return response.rows.map((row: any) => row.doc)
+}
+
+export const backfill = async (
+  globalDb: any,
+  account: CloudAccount | undefined
+) => {
+  const users = await getUsers(globalDb)
+
+  for (const user of users) {
+    let timestamp: string | number = DEFAULT_TIMESTAMP
+    if (user.createdAt) {
+      timestamp = user.createdAt
+    }
+    await events.identification.identifyUser(user, account, timestamp)
+    await events.user.created(user, timestamp)
+
+    if (user.admin?.global) {
+      await events.user.permissionAdminAssigned(user, timestamp)
+    }
+
+    if (user.builder?.global) {
+      await events.user.permissionBuilderAssigned(user, timestamp)
+    }
+
+    if (user.roles) {
+      for (const [appId, role] of Object.entries(user.roles)) {
+        await events.role.assigned(user, role, timestamp)
+      }
+    }
+  }
+
+  return users.length
+}

package/src/migrations/functions/backfill/global.ts

@@ -0,0 +1,194 @@
+import * as users from "./global/users"
+import * as configs from "./global/configs"
+import * as quotas from "./global/quotas"
+import {
+  tenancy,
+  events,
+  migrations,
+  accounts,
+  db as dbUtils,
+} from "@budibase/backend-core"
+import { QuotaUsage } from "@budibase/pro"
+import {
+  CloudAccount,
+  App,
+  TenantBackfillSucceededEvent,
+  Event,
+} from "@budibase/types"
+import env from "../../../environment"
+import { DEFAULT_TIMESTAMP } from "."
+
+const failGraceful = env.SELF_HOSTED && !env.isDev()
+
+const handleError = (e: any, errors?: any) => {
+  if (failGraceful) {
+    if (errors) {
+      errors.push(e)
+    }
+    return
+  }
+  throw e
+}
+
+const formatUsage = (usage: QuotaUsage) => {
+  let maxAutomations = 0
+  let maxQueries = 0
+  let rows = 0
+  let developers = 0
+
+  if (usage) {
+    if (usage.usageQuota) {
+      rows = usage.usageQuota.rows
+      developers = usage.usageQuota.developers
+    }
+
+    if (usage.monthly) {
+      for (const value of Object.values(usage.monthly)) {
+        if (value.automations > maxAutomations) {
+          maxAutomations = value.automations
+        }
+        if (value.queries > maxQueries) {
+          maxQueries = value.queries
+        }
+      }
+    }
+  }
+
+  return {
+    maxAutomations,
+    maxQueries,
+    rows,
+    developers,
+  }
+}
+
+const EVENTS = [
+  Event.EMAIL_SMTP_CREATED,
+  Event.AUTH_SSO_CREATED,
+  Event.AUTH_SSO_ACTIVATED,
+  Event.ORG_NAME_UPDATED,
+  Event.ORG_LOGO_UPDATED,
+  Event.ORG_PLATFORM_URL_UPDATED,
+  Event.USER_CREATED,
+  Event.USER_PERMISSION_ADMIN_ASSIGNED,
+  Event.USER_PERMISSION_BUILDER_ASSIGNED,
+  Event.ROLE_ASSIGNED,
+  Event.ROWS_CREATED,
+  Event.QUERIES_RUN,
+  Event.AUTOMATIONS_RUN,
+]
+
+/**
+ * Date:
+ * May 2022
+ *
+ * Description:
+ * Backfill global events.
+ */
+
+export const run = async (db: any) => {
+  try {
+    const tenantId = tenancy.getTenantId()
+    let timestamp: string | number = DEFAULT_TIMESTAMP
+
+    const totals: any = {}
+    const errors: any = []
+
+    try {
+      const installTimestamp = await getInstallTimestamp(db)
+      if (installTimestamp) {
+        timestamp = installTimestamp
+      }
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    let account: CloudAccount | undefined
+    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+      account = await accounts.getAccountByTenantId(tenantId)
+    }
+
+    try {
+      await events.identification.identifyTenantGroup(
+        tenantId,
+        account,
+        timestamp
+      )
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    // tell the event pipeline to start caching
+    // events for this tenant
+    await events.backfillCache.start(EVENTS)
+
+    try {
+      await configs.backfill(db, timestamp)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    try {
+      totals.users = await users.backfill(db, account)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    try {
+      const allApps: App[] = await dbUtils.getAllApps({ dev: true })
+      totals.apps = allApps.length
+
+      totals.usage = await quotas.backfill(allApps)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    const properties: TenantBackfillSucceededEvent = {
+      apps: totals.apps,
+      users: totals.users,
+      ...formatUsage(totals.usage),
+      usage: totals.usage,
+    }
+
+    if (errors.length) {
+      properties.errors = errors.map((e: any) =>
+        JSON.stringify(e, Object.getOwnPropertyNames(e))
+      )
+      properties.errorCount = errors.length
+    } else {
+      properties.errorCount = 0
+    }
+
+    await events.backfill.tenantSucceeded(properties)
+    // tell the event pipeline to stop caching events for this tenant
+    await events.backfillCache.end()
+  } catch (e) {
+    handleError(e)
+    await events.backfill.tenantFailed(e)
+  }
+}
+
+export const isComplete = async (): Promise<boolean> => {
+  const globalDb = tenancy.getGlobalDB()
+  const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
+  return !!migrationsDoc.event_global_backfill
+}
+
+export const getInstallTimestamp = async (
+  globalDb: any
+): Promise<number | undefined> => {
+  const allUsers = await users.getUsers(globalDb)
+
+  // get the oldest user timestamp
+  const timestamps = allUsers
+    .map(user => user.createdAt)
+    .filter(timestamp => !!timestamp)
+    .sort(
+      (a, b) =>
+        new Date(a as number).getTime() - new Date(b as number).getTime()
+    )
+
+  if (timestamps.length) {
+    return timestamps[0]
+  }
+}

package/src/migrations/functions/backfill/index.ts

@@ -0,0 +1,7 @@
+export * as app from "./app"
+export * as global from "./global"
+export * as installation from "./installation"
+
+// historical events are free in posthog - make sure we default to a
+// historical time if no other can be found
+export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()

package/src/migrations/functions/backfill/installation.ts

@@ -0,0 +1,51 @@
+import { DEFAULT_TIMESTAMP } from "./index"
+import { events, tenancy, installation } from "@budibase/backend-core"
+import { Installation } from "@budibase/types"
+import * as global from "./global"
+import env from "../../../environment"
+
+const failGraceful = env.SELF_HOSTED
+
+const handleError = (e: any, errors?: any) => {
+  if (failGraceful) {
+    if (errors) {
+      errors.push(e)
+    }
+    return
+  }
+  throw e
+}
+
+/**
+ * Date:
+ * May 2022
+ *
+ * Description:
+ * Backfill installation events.
+ */
+
+export const run = async () => {
+  try {
+    // need to use the default tenant to try to get the installation time
+    await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
+      const db = tenancy.getGlobalDB()
+      let timestamp: string | number = DEFAULT_TIMESTAMP
+
+      const installTimestamp = await global.getInstallTimestamp(db)
+      if (installTimestamp) {
+        timestamp = installTimestamp
+      }
+
+      const install: Installation = await installation.getInstall()
+      await events.identification.identifyInstallationGroup(
+        install.installId,
+        timestamp
+      )
+    })
+    await events.backfill.installationSucceeded()
+    throw new Error("fail")
+  } catch (e) {
+    handleError(e)
+    await events.backfill.installationFailed(e)
+  }
+}