@budibase/server 1.0.200-alpha.3 → 1.0.200

package/src/migrations/functions/backfill/app/tables.ts

@@ -1,36 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getTableParams } from "../../../../db/utils"
-import { Table } from "@budibase/types"
-
-const getTables = async (appDb: any): Promise<Table[]> => {
-  const response = await appDb.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const tables = await getTables(appDb)
-
-  for (const table of tables) {
-    await events.table.created(table, timestamp)
-
-    if (table.views) {
-      for (const view of Object.values(table.views)) {
-        await events.view.created(view, timestamp)
-
-        if (view.calculation) {
-          await events.view.calculationCreated(view, timestamp)
-        }
-
-        if (view.filters?.length) {
-          await events.view.filterCreated(view, timestamp)
-        }
-      }
-    }
-  }
-
-  return tables.length
-}

package/src/migrations/functions/backfill/app.ts

@@ -1,148 +0,0 @@
-import * as automations from "./app/automations"
-import * as datasources from "./app/datasources"
-import * as layouts from "./app/layouts"
-import * as queries from "./app/queries"
-import * as roles from "./app/roles"
-import * as tables from "./app/tables"
-import * as screens from "./app/screens"
-import * as global from "./global"
-import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
-import { db as dbUtils, events } from "@budibase/backend-core"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  throw e
-}
-
-const EVENTS = [
-  Event.AUTOMATION_CREATED,
-  Event.AUTOMATION_STEP_CREATED,
-  Event.DATASOURCE_CREATED,
-  Event.LAYOUT_CREATED,
-  Event.QUERY_CREATED,
-  Event.ROLE_CREATED,
-  Event.SCREEN_CREATED,
-  Event.TABLE_CREATED,
-  Event.VIEW_CREATED,
-  Event.VIEW_CALCULATION_CREATED,
-  Event.VIEW_FILTER_CREATED,
-  Event.APP_PUBLISHED,
-  Event.APP_CREATED,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill app events.
- */
-
-export const run = async (appDb: any) => {
-  try {
-    if (await global.isComplete()) {
-      // make sure new apps aren't backfilled
-      // return if the global migration for this tenant is complete
-      // which runs after the app migrations
-      return
-    }
-
-    // tell the event pipeline to start caching
-    // events for this tenant
-    await events.backfillCache.start(EVENTS)
-
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-    const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA)
-    if (app.createdAt) {
-      timestamp = app.createdAt as string
-    }
-
-    if (dbUtils.isProdAppID(app.appId)) {
-      await events.app.published(app, timestamp)
-    }
-
-    const totals: any = {}
-    const errors: any = []
-
-    if (dbUtils.isDevAppID(app.appId)) {
-      await events.app.created(app, timestamp)
-      try {
-        totals.automations = await automations.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.datasources = await datasources.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.layouts = await layouts.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.queries = await queries.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.roles = await roles.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.screens = await screens.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-
-      try {
-        totals.tables = await tables.backfill(appDb, timestamp)
-      } catch (e) {
-        handleError(e, errors)
-      }
-    }
-
-    const properties: AppBackfillSucceededEvent = {
-      appId: app.appId,
-      automations: totals.automations,
-      datasources: totals.datasources,
-      layouts: totals.layouts,
-      queries: totals.queries,
-      roles: totals.roles,
-      tables: totals.tables,
-      screens: totals.screens,
-    }
-
-    if (errors.length) {
-      properties.errors = errors.map((e: any) =>
-        JSON.stringify(e, Object.getOwnPropertyNames(e))
-      )
-      properties.errorCount = errors.length
-    } else {
-      properties.errorCount = 0
-    }
-
-    await events.backfill.appSucceeded(properties)
-    // tell the event pipeline to stop caching events for this tenant
-    await events.backfillCache.end()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.appFailed(e)
-  }
-}

package/src/migrations/functions/backfill/global/configs.ts

@@ -1,66 +0,0 @@
-import { events, db as dbUtils } from "@budibase/backend-core"
-import {
-  Config,
-  isSMTPConfig,
-  isGoogleConfig,
-  isOIDCConfig,
-  isSettingsConfig,
-} from "@budibase/types"
-import env from "./../../../../environment"
-
-const getConfigs = async (globalDb: any): Promise<Config[]> => {
-  const response = await globalDb.allDocs(
-    dbUtils.getConfigParams(
-      {},
-      {
-        include_docs: true,
-      }
-    )
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (
-  globalDb: any,
-  timestamp: string | number | undefined
-) => {
-  const configs = await getConfigs(globalDb)
-
-  for (const config of configs) {
-    if (isSMTPConfig(config)) {
-      await events.email.SMTPCreated(timestamp)
-    }
-    if (isGoogleConfig(config)) {
-      await events.auth.SSOCreated("google", timestamp)
-      if (config.config.activated) {
-        await events.auth.SSOActivated("google", timestamp)
-      }
-    }
-    if (isOIDCConfig(config)) {
-      await events.auth.SSOCreated("oidc", timestamp)
-      if (config.config.configs[0].activated) {
-        await events.auth.SSOActivated("oidc", timestamp)
-      }
-    }
-    if (isSettingsConfig(config)) {
-      const company = config.config.company
-      if (company && company !== "Budibase") {
-        await events.org.nameUpdated(timestamp)
-      }
-
-      const logoUrl = config.config.logoUrl
-      if (logoUrl) {
-        await events.org.logoUpdated(timestamp)
-      }
-
-      const platformUrl = config.config.platformUrl
-      if (
-        platformUrl &&
-        platformUrl !== "http://localhost:10000" &&
-        env.SELF_HOSTED
-      ) {
-        await events.org.platformURLUpdated(timestamp)
-      }
-    }
-  }
-}

package/src/migrations/functions/backfill/global/quotas.ts

@@ -1,60 +0,0 @@
-import { DEFAULT_TIMESTAMP } from "./../index"
-import { events } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { App } from "@budibase/types"
-
-const getOldestCreatedAt = (allApps: App[]): string | undefined => {
-  const timestamps = allApps
-    .filter(app => !!app.createdAt)
-    .map(app => app.createdAt as string)
-    .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
-
-  if (timestamps.length) {
-    return timestamps[0]
-  }
-}
-
-const getMonthTimestamp = (monthString: string): number => {
-  const parts = monthString.split("-")
-  const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
-  const year = parseInt(parts[1])
-
-  // using 0 as the day in next month gives us last day in previous month
-  const date = new Date(year, month + 1, 0).getTime()
-  const now = new Date().getTime()
-
-  if (date > now) {
-    return now
-  } else {
-    return date
-  }
-}
-
-export const backfill = async (allApps: App[]) => {
-  const usage = await quotas.getQuotaUsage()
-
-  const rows = usage.usageQuota.rows
-  let timestamp: string | number = DEFAULT_TIMESTAMP
-
-  const oldestAppTimestamp = getOldestCreatedAt(allApps)
-  if (oldestAppTimestamp) {
-    timestamp = oldestAppTimestamp
-  }
-
-  await events.rows.created(rows, timestamp)
-
-  for (const [monthString, quotas] of Object.entries(usage.monthly)) {
-    if (monthString === "current") {
-      continue
-    }
-    const monthTimestamp = getMonthTimestamp(monthString)
-
-    const queries = quotas.queries
-    await events.query.run(queries, monthTimestamp)
-
-    const automations = quotas.automations
-    await events.automation.run(automations, monthTimestamp)
-  }
-
-  return usage
-}

package/src/migrations/functions/backfill/global/users.ts

@@ -1,49 +0,0 @@
-import { events, db as dbUtils } from "@budibase/backend-core"
-import { User, CloudAccount } from "@budibase/types"
-import { DEFAULT_TIMESTAMP } from ".."
-
-// manually define user doc params - normally server doesn't read users from the db
-const getUserParams = (props: any) => {
-  return dbUtils.getDocParams(dbUtils.DocumentTypes.USER, null, props)
-}
-
-export const getUsers = async (globalDb: any): Promise<User[]> => {
-  const response = await globalDb.allDocs(
-    getUserParams({
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (
-  globalDb: any,
-  account: CloudAccount | undefined
-) => {
-  const users = await getUsers(globalDb)
-
-  for (const user of users) {
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-    if (user.createdAt) {
-      timestamp = user.createdAt
-    }
-    await events.identification.identifyUser(user, account, timestamp)
-    await events.user.created(user, timestamp)
-
-    if (user.admin?.global) {
-      await events.user.permissionAdminAssigned(user, timestamp)
-    }
-
-    if (user.builder?.global) {
-      await events.user.permissionBuilderAssigned(user, timestamp)
-    }
-
-    if (user.roles) {
-      for (const [appId, role] of Object.entries(user.roles)) {
-        await events.role.assigned(user, role, timestamp)
-      }
-    }
-  }
-
-  return users.length
-}

package/src/migrations/functions/backfill/global.ts

@@ -1,194 +0,0 @@
-import * as users from "./global/users"
-import * as configs from "./global/configs"
-import * as quotas from "./global/quotas"
-import {
-  tenancy,
-  events,
-  migrations,
-  accounts,
-  db as dbUtils,
-} from "@budibase/backend-core"
-import { QuotaUsage } from "@budibase/pro"
-import {
-  CloudAccount,
-  App,
-  TenantBackfillSucceededEvent,
-  Event,
-} from "@budibase/types"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  throw e
-}
-
-const formatUsage = (usage: QuotaUsage) => {
-  let maxAutomations = 0
-  let maxQueries = 0
-  let rows = 0
-  let developers = 0
-
-  if (usage) {
-    if (usage.usageQuota) {
-      rows = usage.usageQuota.rows
-      developers = usage.usageQuota.developers
-    }
-
-    if (usage.monthly) {
-      for (const value of Object.values(usage.monthly)) {
-        if (value.automations > maxAutomations) {
-          maxAutomations = value.automations
-        }
-        if (value.queries > maxQueries) {
-          maxQueries = value.queries
-        }
-      }
-    }
-  }
-
-  return {
-    maxAutomations,
-    maxQueries,
-    rows,
-    developers,
-  }
-}
-
-const EVENTS = [
-  Event.EMAIL_SMTP_CREATED,
-  Event.AUTH_SSO_CREATED,
-  Event.AUTH_SSO_ACTIVATED,
-  Event.ORG_NAME_UPDATED,
-  Event.ORG_LOGO_UPDATED,
-  Event.ORG_PLATFORM_URL_UPDATED,
-  Event.USER_CREATED,
-  Event.USER_PERMISSION_ADMIN_ASSIGNED,
-  Event.USER_PERMISSION_BUILDER_ASSIGNED,
-  Event.ROLE_ASSIGNED,
-  Event.ROWS_CREATED,
-  Event.QUERIES_RUN,
-  Event.AUTOMATIONS_RUN,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill global events.
- */
-
-export const run = async (db: any) => {
-  try {
-    const tenantId = tenancy.getTenantId()
-    let timestamp: string | number = DEFAULT_TIMESTAMP
-
-    const totals: any = {}
-    const errors: any = []
-
-    try {
-      const installTimestamp = await getInstallTimestamp(db)
-      if (installTimestamp) {
-        timestamp = installTimestamp
-      }
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    let account: CloudAccount | undefined
-    if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-      account = await accounts.getAccountByTenantId(tenantId)
-    }
-
-    try {
-      await events.identification.identifyTenantGroup(
-        tenantId,
-        account,
-        timestamp
-      )
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    // tell the event pipeline to start caching
-    // events for this tenant
-    await events.backfillCache.start(EVENTS)
-
-    try {
-      await configs.backfill(db, timestamp)
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    try {
-      totals.users = await users.backfill(db, account)
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    try {
-      const allApps: App[] = await dbUtils.getAllApps({ dev: true })
-      totals.apps = allApps.length
-
-      totals.usage = await quotas.backfill(allApps)
-    } catch (e) {
-      handleError(e, errors)
-    }
-
-    const properties: TenantBackfillSucceededEvent = {
-      apps: totals.apps,
-      users: totals.users,
-      ...formatUsage(totals.usage),
-      usage: totals.usage,
-    }
-
-    if (errors.length) {
-      properties.errors = errors.map((e: any) =>
-        JSON.stringify(e, Object.getOwnPropertyNames(e))
-      )
-      properties.errorCount = errors.length
-    } else {
-      properties.errorCount = 0
-    }
-
-    await events.backfill.tenantSucceeded(properties)
-    // tell the event pipeline to stop caching events for this tenant
-    await events.backfillCache.end()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.tenantFailed(e)
-  }
-}
-
-export const isComplete = async (): Promise<boolean> => {
-  const globalDb = tenancy.getGlobalDB()
-  const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
-  return !!migrationsDoc.event_global_backfill
-}
-
-export const getInstallTimestamp = async (
-  globalDb: any
-): Promise<number | undefined> => {
-  const allUsers = await users.getUsers(globalDb)
-
-  // get the oldest user timestamp
-  const timestamps = allUsers
-    .map(user => user.createdAt)
-    .filter(timestamp => !!timestamp)
-    .sort(
-      (a, b) =>
-        new Date(a as number).getTime() - new Date(b as number).getTime()
-    )
-
-  if (timestamps.length) {
-    return timestamps[0]
-  }
-}

package/src/migrations/functions/backfill/index.ts

@@ -1,7 +0,0 @@
-export * as app from "./app"
-export * as global from "./global"
-export * as installation from "./installation"
-
-// historical events are free in posthog - make sure we default to a
-// historical time if no other can be found
-export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()

package/src/migrations/functions/backfill/installation.ts

@@ -1,50 +0,0 @@
-import { DEFAULT_TIMESTAMP } from "./index"
-import { events, tenancy, installation } from "@budibase/backend-core"
-import { Installation } from "@budibase/types"
-import * as global from "./global"
-import env from "../../../environment"
-
-const failGraceful = env.SELF_HOSTED
-
-const handleError = (e: any, errors?: any) => {
-  if (failGraceful) {
-    if (errors) {
-      errors.push(e)
-    }
-    return
-  }
-  throw e
-}
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill installation events.
- */
-
-export const run = async () => {
-  try {
-    // need to use the default tenant to try to get the installation time
-    await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
-      const db = tenancy.getGlobalDB()
-      let timestamp: string | number = DEFAULT_TIMESTAMP
-
-      const installTimestamp = await global.getInstallTimestamp(db)
-      if (installTimestamp) {
-        timestamp = installTimestamp
-      }
-
-      const install: Installation = await installation.getInstall()
-      await events.identification.identifyInstallationGroup(
-        install.installId,
-        timestamp
-      )
-    })
-    await events.backfill.installationSucceeded()
-  } catch (e) {
-    handleError(e)
-    await events.backfill.installationFailed(e)
-  }
-}

package/src/migrations/tests/helpers.ts

@@ -1,40 +0,0 @@
-// Mimic configs test configuration from worker, creation configs directly in database
-
-import * as structures from "./structures"
-import { db } from "@budibase/backend-core"
-import { Config } from "@budibase/types"
-
-export const saveSettingsConfig = async (globalDb: any) => {
-  const config = structures.settings()
-  await saveConfig(config, globalDb)
-}
-
-export const saveGoogleConfig = async (globalDb: any) => {
-  const config = structures.google()
-  await saveConfig(config, globalDb)
-}
-
-export const saveOIDCConfig = async (globalDb: any) => {
-  const config = structures.oidc()
-  await saveConfig(config, globalDb)
-}
-
-export const saveSmtpConfig = async (globalDb: any) => {
-  const config = structures.smtp()
-  await saveConfig(config, globalDb)
-}
-
-const saveConfig = async (config: Config, globalDb: any) => {
-  config._id = db.generateConfigID({ type: config.type })
-
-  let response
-  try {
-    response = await globalDb.get(config._id)
-    config._rev = response._rev
-    await globalDb.put(config)
-  } catch (e: any) {
-    if (e.status === 404) {
-      await globalDb.put(config)
-    }
-  }
-}