@budibase/backend-core 2.9.19 → 2.9.21-alpha.0

package/src/events/backfill.ts

@@ -1,183 +0,0 @@
-import {
-  Event,
-  BackfillMetadata,
-  CachedEvent,
-  SSOCreatedEvent,
-  AutomationCreatedEvent,
-  AutomationStepCreatedEvent,
-  DatasourceCreatedEvent,
-  LayoutCreatedEvent,
-  QueryCreatedEvent,
-  RoleCreatedEvent,
-  ScreenCreatedEvent,
-  TableCreatedEvent,
-  ViewCreatedEvent,
-  ViewCalculationCreatedEvent,
-  ViewFilterCreatedEvent,
-  AppPublishedEvent,
-  UserCreatedEvent,
-  RoleAssignedEvent,
-  UserPermissionAssignedEvent,
-  AppCreatedEvent,
-} from "@budibase/types"
-import * as context from "../context"
-import { CacheKey } from "../cache/generic"
-import * as cache from "../cache/generic"
-
-// LIFECYCLE
-
-export const start = async (events: Event[]) => {
-  const metadata: BackfillMetadata = {
-    eventWhitelist: events,
-  }
-  return saveBackfillMetadata(metadata)
-}
-
-export const recordEvent = async (event: Event, properties: any) => {
-  const eventKey = getEventKey(event, properties)
-  // don't use a ttl - cleaned up by migration
-  // don't use tenancy - already in the key
-  await cache.store(eventKey, properties, undefined, { useTenancy: false })
-}
-
-export const end = async () => {
-  await deleteBackfillMetadata()
-  await clearEvents()
-}
-
-// CRUD
-
-const getBackfillMetadata = async (): Promise<BackfillMetadata | null> => {
-  return cache.get(CacheKey.BACKFILL_METADATA)
-}
-
-const saveBackfillMetadata = async (
-  backfill: BackfillMetadata
-): Promise<void> => {
-  // no TTL - deleted by backfill
-  return cache.store(CacheKey.BACKFILL_METADATA, backfill)
-}
-
-const deleteBackfillMetadata = async (): Promise<void> => {
-  await cache.destroy(CacheKey.BACKFILL_METADATA)
-}
-
-const clearEvents = async () => {
-  // wildcard
-  const pattern = getEventKey()
-  const keys = await cache.keys(pattern)
-
-  for (const key of keys) {
-    // delete each key
-    // don't  use tenancy, already in the key
-    await cache.destroy(key, { useTenancy: false })
-  }
-}
-
-// HELPERS
-
-export const isBackfillingEvent = async (event: Event) => {
-  const backfill = await getBackfillMetadata()
-  const events = backfill?.eventWhitelist
-  if (events && events.includes(event)) {
-    return true
-  } else {
-    return false
-  }
-}
-
-export const isAlreadySent = async (event: Event, properties: any) => {
-  const eventKey = getEventKey(event, properties)
-  const cachedEvent: CachedEvent = await cache.get(eventKey, {
-    useTenancy: false,
-  })
-  return !!cachedEvent
-}
-
-const CUSTOM_PROPERTY_SUFFIX: any = {
-  // APP EVENTS
-  [Event.AUTOMATION_CREATED]: (properties: AutomationCreatedEvent) => {
-    return properties.automationId
-  },
-  [Event.AUTOMATION_STEP_CREATED]: (properties: AutomationStepCreatedEvent) => {
-    return properties.stepId
-  },
-  [Event.DATASOURCE_CREATED]: (properties: DatasourceCreatedEvent) => {
-    return properties.datasourceId
-  },
-  [Event.LAYOUT_CREATED]: (properties: LayoutCreatedEvent) => {
-    return properties.layoutId
-  },
-  [Event.QUERY_CREATED]: (properties: QueryCreatedEvent) => {
-    return properties.queryId
-  },
-  [Event.ROLE_CREATED]: (properties: RoleCreatedEvent) => {
-    return properties.roleId
-  },
-  [Event.SCREEN_CREATED]: (properties: ScreenCreatedEvent) => {
-    return properties.screenId
-  },
-  [Event.TABLE_CREATED]: (properties: TableCreatedEvent) => {
-    return properties.tableId
-  },
-  [Event.VIEW_CREATED]: (properties: ViewCreatedEvent) => {
-    return properties.tableId // best uniqueness
-  },
-  [Event.VIEW_CALCULATION_CREATED]: (
-    properties: ViewCalculationCreatedEvent
-  ) => {
-    return properties.tableId // best uniqueness
-  },
-  [Event.VIEW_FILTER_CREATED]: (properties: ViewFilterCreatedEvent) => {
-    return properties.tableId // best uniqueness
-  },
-  [Event.APP_CREATED]: (properties: AppCreatedEvent) => {
-    return properties.appId // best uniqueness
-  },
-  [Event.APP_PUBLISHED]: (properties: AppPublishedEvent) => {
-    return properties.appId // best uniqueness
-  },
-  // GLOBAL EVENTS
-  [Event.AUTH_SSO_CREATED]: (properties: SSOCreatedEvent) => {
-    return properties.type
-  },
-  [Event.AUTH_SSO_ACTIVATED]: (properties: SSOCreatedEvent) => {
-    return properties.type
-  },
-  [Event.USER_CREATED]: (properties: UserCreatedEvent) => {
-    return properties.userId
-  },
-  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
-    properties: UserPermissionAssignedEvent
-  ) => {
-    return properties.userId
-  },
-  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
-    properties: UserPermissionAssignedEvent
-  ) => {
-    return properties.userId
-  },
-  [Event.ROLE_ASSIGNED]: (properties: RoleAssignedEvent) => {
-    return `${properties.roleId}-${properties.userId}`
-  },
-}
-
-const getEventKey = (event?: Event, properties?: any) => {
-  let eventKey: string
-
-  const tenantId = context.getTenantId()
-  if (event) {
-    eventKey = `${CacheKey.EVENTS}:${tenantId}:${event}`
-
-    // use some properties to make the key more unique
-    const custom = CUSTOM_PROPERTY_SUFFIX[event]
-    const suffix = custom ? custom(properties) : undefined
-    if (suffix) {
-      eventKey = `${eventKey}:${suffix}`
-    }
-  } else {
-    eventKey = `${CacheKey.EVENTS}:${tenantId}:*`
-  }
-
-  return eventKey
-}

package/src/events/documentId.ts

@@ -1,56 +0,0 @@
-import {
-  Event,
-  UserCreatedEvent,
-  UserUpdatedEvent,
-  UserDeletedEvent,
-  UserPermissionAssignedEvent,
-  UserPermissionRemovedEvent,
-  GroupCreatedEvent,
-  GroupUpdatedEvent,
-  GroupDeletedEvent,
-  GroupUsersAddedEvent,
-  GroupUsersDeletedEvent,
-  GroupPermissionsEditedEvent,
-} from "@budibase/types"
-
-const getEventProperties: Record<
-  string,
-  (properties: any) => string | undefined
-> = {
-  [Event.USER_CREATED]: (properties: UserCreatedEvent) => properties.userId,
-  [Event.USER_UPDATED]: (properties: UserUpdatedEvent) => properties.userId,
-  [Event.USER_DELETED]: (properties: UserDeletedEvent) => properties.userId,
-  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
-    properties: UserPermissionAssignedEvent
-  ) => properties.userId,
-  [Event.USER_PERMISSION_ADMIN_REMOVED]: (
-    properties: UserPermissionRemovedEvent
-  ) => properties.userId,
-  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
-    properties: UserPermissionAssignedEvent
-  ) => properties.userId,
-  [Event.USER_PERMISSION_BUILDER_REMOVED]: (
-    properties: UserPermissionRemovedEvent
-  ) => properties.userId,
-  [Event.USER_GROUP_CREATED]: (properties: GroupCreatedEvent) =>
-    properties.groupId,
-  [Event.USER_GROUP_UPDATED]: (properties: GroupUpdatedEvent) =>
-    properties.groupId,
-  [Event.USER_GROUP_DELETED]: (properties: GroupDeletedEvent) =>
-    properties.groupId,
-  [Event.USER_GROUP_USERS_ADDED]: (properties: GroupUsersAddedEvent) =>
-    properties.groupId,
-  [Event.USER_GROUP_USERS_REMOVED]: (properties: GroupUsersDeletedEvent) =>
-    properties.groupId,
-  [Event.USER_GROUP_PERMISSIONS_EDITED]: (
-    properties: GroupPermissionsEditedEvent
-  ) => properties.groupId,
-}
-
-export function getDocumentId(event: Event, properties: any) {
-  const extractor = getEventProperties[event]
-  if (!extractor) {
-    throw new Error("Event does not have a method of document ID extraction")
-  }
-  return extractor(properties)
-}

package/src/events/events.ts

@@ -1,40 +0,0 @@
-import { Event } from "@budibase/types"
-import { processors } from "./processors"
-import identification from "./identification"
-import * as backfill from "./backfill"
-import { publishAsyncEvent } from "./asyncEvents"
-
-export const publishEvent = async (
-  event: Event,
-  properties: any,
-  timestamp?: string | number
-) => {
-  // in future this should use async events via a distributed queue.
-  const identity = await identification.getCurrentIdentity()
-
-  const backfilling = await backfill.isBackfillingEvent(event)
-  // no backfill - send the event and exit
-  if (!backfilling) {
-    // send off async events if required
-    await publishAsyncEvent({
-      event,
-      identity,
-      properties,
-      timestamp,
-    })
-    // now handle the main sync event processing pipeline
-    await processors.processEvent(event, identity, properties, timestamp)
-    return
-  }
-
-  // backfill active - check if the event has been sent already
-  const alreadySent = await backfill.isAlreadySent(event, properties)
-  if (alreadySent) {
-    // do nothing
-    return
-  } else {
-    // send and record the event
-    await processors.processEvent(event, identity, properties, timestamp)
-    await backfill.recordEvent(event, properties)
-  }
-}

package/src/events/identification.ts

@@ -1,310 +0,0 @@
-import * as context from "../context"
-import * as identityCtx from "../context/identity"
-import env from "../environment"
-import {
-  Hosting,
-  User,
-  Identity,
-  IdentityType,
-  Account,
-  isCloudAccount,
-  isSSOAccount,
-  TenantGroup,
-  CloudAccount,
-  UserIdentity,
-  InstallationGroup,
-  UserContext,
-  Group,
-  isSSOUser,
-} from "@budibase/types"
-import { processors } from "./processors"
-import { newid } from "../utils"
-import * as installation from "../installation"
-import * as configs from "../configs"
-import * as users from "../users"
-import { withCache, TTL, CacheKey } from "../cache/generic"
-
-/**
- * An identity can be:
- * - account user (Self host)
- * - budibase user
- * - tenant
- * - installation
- */
-const getCurrentIdentity = async (): Promise<Identity> => {
-  let identityContext = identityCtx.getIdentity()
-  const environment = getDeploymentEnvironment()
-
-  let identityType
-
-  if (!identityContext) {
-    identityType = IdentityType.TENANT
-  } else {
-    identityType = identityContext.type
-  }
-
-  if (identityType === IdentityType.INSTALLATION) {
-    const installationId = await getInstallationId()
-    const hosting = getHostingFromEnv()
-    return {
-      id: formatDistinctId(installationId, identityType),
-      hosting,
-      type: identityType,
-      installationId,
-      environment,
-    }
-  } else if (identityType === IdentityType.TENANT) {
-    const installationId = await getInstallationId()
-    const tenantId = await getEventTenantId(context.getTenantId())
-    const hosting = getHostingFromEnv()
-
-    return {
-      id: formatDistinctId(tenantId, identityType),
-      type: identityType,
-      hosting,
-      installationId,
-      tenantId,
-      realTenantId: context.getTenantId(),
-      environment,
-    }
-  } else if (identityType === IdentityType.USER) {
-    const userContext = identityContext as UserContext
-    const tenantId = await getEventTenantId(context.getTenantId())
-    const installationId = await getInstallationId()
-
-    const account = userContext.account
-    let hosting
-    if (account) {
-      hosting = account.hosting
-    } else {
-      hosting = getHostingFromEnv()
-    }
-
-    return {
-      id: userContext._id,
-      type: identityType,
-      hosting,
-      installationId,
-      tenantId,
-      environment,
-      realTenantId: context.getTenantId(),
-      hostInfo: userContext.hostInfo,
-    }
-  } else {
-    throw new Error("Unknown identity type")
-  }
-}
-
-const identifyInstallationGroup = async (
-  installId: string,
-  timestamp?: string | number
-): Promise<void> => {
-  const id = installId
-  const type = IdentityType.INSTALLATION
-  const hosting = getHostingFromEnv()
-  const version = env.VERSION
-  const environment = getDeploymentEnvironment()
-
-  const group: InstallationGroup = {
-    id,
-    type,
-    hosting,
-    version,
-    environment,
-  }
-
-  await identifyGroup(group, timestamp)
-  // need to create a normal identity for the group to be able to query it globally
-  // match the posthog syntax to link this identity to the empty auto generated one
-  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
-}
-
-const identifyTenantGroup = async (
-  tenantId: string,
-  account: Account | undefined,
-  timestamp?: string | number
-): Promise<void> => {
-  const id = await getEventTenantId(tenantId)
-  const type = IdentityType.TENANT
-  const installationId = await getInstallationId()
-  const environment = getDeploymentEnvironment()
-
-  let hosting: Hosting
-  let profession: string | undefined
-  let companySize: string | undefined
-
-  if (account) {
-    profession = account.profession
-    companySize = account.size
-    hosting = account.hosting
-  } else {
-    hosting = getHostingFromEnv()
-  }
-
-  const group: TenantGroup = {
-    id,
-    type,
-    hosting,
-    environment,
-    installationId,
-    profession,
-    companySize,
-  }
-
-  await identifyGroup(group, timestamp)
-  // need to create a normal identity for the group to be able to query it globally
-  // match the posthog syntax to link this identity to the auto generated one
-  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
-}
-
-const identifyUser = async (
-  user: User,
-  account: CloudAccount | undefined,
-  timestamp?: string | number
-) => {
-  const id = user._id as string
-  const tenantId = await getEventTenantId(user.tenantId)
-  const type = IdentityType.USER
-  let builder = users.hasBuilderPermissions(user)
-  let admin = users.hasAdminPermissions(user)
-  let providerType
-  if (isSSOUser(user)) {
-    providerType = user.providerType
-  }
-  const accountHolder = account?.budibaseUserId === user._id || false
-  const verified =
-    account && account?.budibaseUserId === user._id ? account.verified : false
-  const installationId = await getInstallationId()
-  const hosting = account ? account.hosting : getHostingFromEnv()
-  const environment = getDeploymentEnvironment()
-
-  const identity: UserIdentity = {
-    id,
-    type,
-    hosting,
-    installationId,
-    tenantId,
-    verified,
-    accountHolder,
-    providerType,
-    builder,
-    admin,
-    environment,
-  }
-
-  await identify(identity, timestamp)
-}
-
-const identifyAccount = async (account: Account) => {
-  let id = account.accountId
-  const tenantId = account.tenantId
-  let type = IdentityType.USER
-  let providerType = isSSOAccount(account) ? account.providerType : undefined
-  const verified = account.verified
-  const accountHolder = true
-  const hosting = account.hosting
-  const installationId = await getInstallationId()
-  const environment = getDeploymentEnvironment()
-
-  if (isCloudAccount(account)) {
-    if (account.budibaseUserId) {
-      // use the budibase user as the id if set
-      id = account.budibaseUserId
-    }
-  }
-
-  const identity: UserIdentity = {
-    id,
-    type,
-    hosting,
-    installationId,
-    tenantId,
-    providerType,
-    verified,
-    accountHolder,
-    environment,
-  }
-
-  await identify(identity)
-}
-
-const identify = async (identity: Identity, timestamp?: string | number) => {
-  await processors.identify(identity, timestamp)
-}
-
-const identifyGroup = async (group: Group, timestamp?: string | number) => {
-  await processors.identifyGroup(group, timestamp)
-}
-
-const getDeploymentEnvironment = () => {
-  if (env.isDev()) {
-    return "development"
-  } else {
-    return env.DEPLOYMENT_ENVIRONMENT
-  }
-}
-
-const getHostingFromEnv = () => {
-  return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
-}
-
-const getInstallationId = async () => {
-  if (isAccountPortal()) {
-    return "account-portal"
-  }
-  const install = await installation.getInstall()
-  return install.installId
-}
-
-const getEventTenantId = async (tenantId: string): Promise<string> => {
-  if (env.SELF_HOSTED) {
-    return getUniqueTenantId(tenantId)
-  } else {
-    // tenant id's in the cloud are already unique
-    return tenantId
-  }
-}
-
-export const getUniqueTenantId = async (tenantId: string): Promise<string> => {
-  // make sure this tenantId always matches the tenantId in context
-  return context.doInTenant(tenantId, () => {
-    return withCache(CacheKey.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
-      const db = context.getGlobalDB()
-      const config = await configs.getSettingsConfigDoc()
-
-      let uniqueTenantId: string
-      if (config.config.uniqueTenantId) {
-        return config.config.uniqueTenantId
-      } else {
-        uniqueTenantId = `${newid()}_${tenantId}`
-        config.config.uniqueTenantId = uniqueTenantId
-        await db.put(config)
-        return uniqueTenantId
-      }
-    })
-  })
-}
-
-const isAccountPortal = () => {
-  return env.SERVICE === "account-portal"
-}
-
-const formatDistinctId = (id: string, type: IdentityType) => {
-  if (type === IdentityType.INSTALLATION || type === IdentityType.TENANT) {
-    return `$${type}_${id}`
-  } else {
-    return id
-  }
-}
-
-export default {
-  getCurrentIdentity,
-  identifyInstallationGroup,
-  identifyTenantGroup,
-  identifyUser,
-  identifyAccount,
-  identify,
-  identifyGroup,
-  getInstallationId,
-  getUniqueTenantId,
-}

package/src/events/index.ts

@@ -1,14 +0,0 @@
-export * from "./publishers"
-export * as processors from "./processors"
-export * as analytics from "./analytics"
-export { default as identification } from "./identification"
-export * as backfillCache from "./backfill"
-
-import { processors } from "./processors"
-
-export function initAsyncEvents() {}
-
-export const shutdown = () => {
-  processors.shutdown()
-  console.log("Events shutdown")
-}

package/src/events/processors/AnalyticsProcessor.ts

@@ -1,64 +0,0 @@
-import { Event, Identity, Group, IdentityType } from "@budibase/types"
-import { EventProcessor } from "./types"
-import env from "../../environment"
-import * as analytics from "../analytics"
-import PosthogProcessor from "./posthog"
-
-/**
- * Events that are always captured.
- */
-const EVENT_WHITELIST = [
-  Event.INSTALLATION_VERSION_UPGRADED,
-  Event.INSTALLATION_VERSION_DOWNGRADED,
-]
-const IDENTITY_WHITELIST = [IdentityType.INSTALLATION, IdentityType.TENANT]
-
-export default class AnalyticsProcessor implements EventProcessor {
-  posthog: PosthogProcessor | undefined
-
-  constructor() {
-    if (env.POSTHOG_TOKEN && !env.isTest()) {
-      this.posthog = new PosthogProcessor(env.POSTHOG_TOKEN)
-    }
-  }
-
-  async processEvent(
-    event: Event,
-    identity: Identity,
-    properties: any,
-    timestamp?: string | number
-  ): Promise<void> {
-    if (!EVENT_WHITELIST.includes(event) && !(await analytics.enabled())) {
-      return
-    }
-    if (this.posthog) {
-      await this.posthog.processEvent(event, identity, properties, timestamp)
-    }
-  }
-
-  async identify(identity: Identity, timestamp?: string | number) {
-    // Group indentifications (tenant and installation) always on
-    if (
-      !IDENTITY_WHITELIST.includes(identity.type) &&
-      !(await analytics.enabled())
-    ) {
-      return
-    }
-    if (this.posthog) {
-      await this.posthog.identify(identity, timestamp)
-    }
-  }
-
-  async identifyGroup(group: Group, timestamp?: string | number) {
-    // Group indentifications (tenant and installation) always on
-    if (this.posthog) {
-      await this.posthog.identifyGroup(group, timestamp)
-    }
-  }
-
-  shutdown() {
-    if (this.posthog) {
-      this.posthog.shutdown()
-    }
-  }
-}