@budibase/backend-core 2.9.19 → 2.9.21-alpha.0

package/src/events/processors/AuditLogsProcessor.ts

@@ -1,93 +0,0 @@
-import {
-  Event,
-  Identity,
-  Group,
-  IdentityType,
-  AuditLogQueueEvent,
-  AuditLogFn,
-  HostInfo,
-} from "@budibase/types"
-import { EventProcessor } from "./types"
-import { getAppId, doInTenant, getTenantId } from "../../context"
-import BullQueue from "bull"
-import { createQueue, JobQueue } from "../../queue"
-import { isAudited } from "../../utils"
-import env from "../../environment"
-
-export default class AuditLogsProcessor implements EventProcessor {
-  static auditLogsEnabled = false
-  static auditLogQueue: BullQueue.Queue<AuditLogQueueEvent>
-
-  // can't use constructor as need to return promise
-  static init(fn: AuditLogFn) {
-    AuditLogsProcessor.auditLogsEnabled = true
-    const writeAuditLogs = fn
-    AuditLogsProcessor.auditLogQueue = createQueue<AuditLogQueueEvent>(
-      JobQueue.AUDIT_LOG
-    )
-    return AuditLogsProcessor.auditLogQueue.process(async job => {
-      return doInTenant(job.data.tenantId, async () => {
-        let properties = job.data.properties
-        if (properties.audited) {
-          properties = {
-            ...properties,
-            ...properties.audited,
-          }
-          delete properties.audited
-        }
-
-        // this feature is disabled by default due to privacy requirements
-        // in some countries - available as env var in-case it is desired
-        // in self host deployments
-        let hostInfo: HostInfo | undefined = {}
-        if (env.ENABLE_AUDIT_LOG_IP_ADDR) {
-          hostInfo = job.data.opts.hostInfo
-        }
-
-        await writeAuditLogs(job.data.event, properties, {
-          userId: job.data.opts.userId,
-          timestamp: job.data.opts.timestamp,
-          appId: job.data.opts.appId,
-          hostInfo,
-        })
-      })
-    })
-  }
-
-  async processEvent(
-    event: Event,
-    identity: Identity,
-    properties: any,
-    timestamp?: string
-  ): Promise<void> {
-    if (AuditLogsProcessor.auditLogsEnabled && isAudited(event)) {
-      // only audit log actual events, don't include backfills
-      const userId =
-        identity.type === IdentityType.USER ? identity.id : undefined
-      // add to the event queue, rather than just writing immediately
-      await AuditLogsProcessor.auditLogQueue.add({
-        event,
-        properties,
-        opts: {
-          userId,
-          timestamp,
-          appId: getAppId(),
-          hostInfo: identity.hostInfo,
-        },
-        tenantId: getTenantId(),
-      })
-    }
-  }
-
-  async identify(identity: Identity, timestamp?: string | number) {
-    // no-op
-  }
-
-  async identifyGroup(group: Group, timestamp?: string | number) {
-    // no-op
-  }
-
-  shutdown(): void {
-    AuditLogsProcessor.auditLogQueue?.close()
-  }
-}

package/src/events/processors/LoggingProcessor.ts

@@ -1,37 +0,0 @@
-import { Event, Identity, Group } from "@budibase/types"
-import { EventProcessor } from "./types"
-import env from "../../environment"
-
-const skipLogging = env.SELF_HOSTED && !env.isDev()
-
-export default class LoggingProcessor implements EventProcessor {
-  async processEvent(
-    event: Event,
-    identity: Identity,
-    properties: any,
-    timestamp?: string
-  ): Promise<void> {
-    if (skipLogging) {
-      return
-    }
-    console.log(`[audit] [identityType=${identity.type}] ${event}`, properties)
-  }
-
-  async identify(identity: Identity, timestamp?: string | number) {
-    if (skipLogging) {
-      return
-    }
-    console.log(`[audit] identified`, identity)
-  }
-
-  async identifyGroup(group: Group, timestamp?: string | number) {
-    if (skipLogging) {
-      return
-    }
-    console.log(`[audit] group identified`, group)
-  }
-
-  shutdown(): void {
-    // no-op
-  }
-}

package/src/events/processors/Processors.ts

@@ -1,52 +0,0 @@
-import { Event, Identity, Group } from "@budibase/types"
-import { EventProcessor } from "./types"
-
-export default class Processor implements EventProcessor {
-  initialised: boolean = false
-  processors: EventProcessor[] = []
-
-  constructor(processors: EventProcessor[]) {
-    this.processors = processors
-  }
-
-  async processEvent(
-    event: Event,
-    identity: Identity,
-    properties: any,
-    timestamp?: string | number
-  ): Promise<void> {
-    for (const eventProcessor of this.processors) {
-      await eventProcessor.processEvent(event, identity, properties, timestamp)
-    }
-  }
-
-  async identify(
-    identity: Identity,
-    timestamp?: string | number
-  ): Promise<void> {
-    for (const eventProcessor of this.processors) {
-      if (eventProcessor.identify) {
-        await eventProcessor.identify(identity, timestamp)
-      }
-    }
-  }
-
-  async identifyGroup(
-    identity: Group,
-    timestamp?: string | number
-  ): Promise<void> {
-    for (const eventProcessor of this.processors) {
-      if (eventProcessor.identifyGroup) {
-        await eventProcessor.identifyGroup(identity, timestamp)
-      }
-    }
-  }
-
-  shutdown() {
-    for (const eventProcessor of this.processors) {
-      if (eventProcessor.shutdown) {
-        eventProcessor.shutdown()
-      }
-    }
-  }
-}

package/src/events/processors/async/DocumentUpdateProcessor.ts

@@ -1,43 +0,0 @@
-import { EventProcessor } from "../types"
-import { Event, Identity, DocUpdateEvent } from "@budibase/types"
-import { doInTenant } from "../../../context"
-import { getDocumentId } from "../../documentId"
-import { shutdown } from "../../asyncEvents"
-
-export type Processor = (update: DocUpdateEvent) => Promise<void>
-export type ProcessorMap = { events: Event[]; processor: Processor }[]
-
-export default class DocumentUpdateProcessor implements EventProcessor {
-  processors: ProcessorMap = []
-
-  constructor(processors: ProcessorMap) {
-    this.processors = processors
-  }
-
-  async processEvent(
-    event: Event,
-    identity: Identity,
-    properties: any,
-    timestamp?: string | number
-  ) {
-    const tenantId = identity.realTenantId
-    const docId = getDocumentId(event, properties)
-    if (!tenantId || !docId) {
-      return
-    }
-    for (let { events, processor } of this.processors) {
-      if (events.includes(event)) {
-        await doInTenant(tenantId, async () => {
-          await processor({
-            id: docId,
-            tenantId,
-          })
-        })
-      }
-    }
-  }
-
-  shutdown() {
-    return shutdown()
-  }
-}

package/src/events/processors/index.ts

@@ -1,19 +0,0 @@
-import AnalyticsProcessor from "./AnalyticsProcessor"
-import LoggingProcessor from "./LoggingProcessor"
-import AuditLogsProcessor from "./AuditLogsProcessor"
-import Processors from "./Processors"
-import { AuditLogFn } from "@budibase/types"
-
-export const analyticsProcessor = new AnalyticsProcessor()
-const loggingProcessor = new LoggingProcessor()
-const auditLogsProcessor = new AuditLogsProcessor()
-
-export function init(auditingFn: AuditLogFn) {
-  return AuditLogsProcessor.init(auditingFn)
-}
-
-export const processors = new Processors([
-  analyticsProcessor,
-  loggingProcessor,
-  auditLogsProcessor,
-])

package/src/events/processors/posthog/PosthogProcessor.ts

@@ -1,118 +0,0 @@
-import PostHog from "posthog-node"
-import { Event, Identity, Group, BaseEvent } from "@budibase/types"
-import { EventProcessor } from "../types"
-import env from "../../../environment"
-import * as context from "../../../context"
-import * as rateLimiting from "./rateLimiting"
-
-const EXCLUDED_EVENTS: Event[] = [
-  Event.USER_UPDATED,
-  Event.EMAIL_SMTP_UPDATED,
-  Event.AUTH_SSO_UPDATED,
-  Event.APP_UPDATED,
-  Event.ROLE_UPDATED,
-  Event.DATASOURCE_UPDATED,
-  Event.QUERY_UPDATED,
-  Event.TABLE_UPDATED,
-  Event.VIEW_UPDATED,
-  Event.VIEW_FILTER_UPDATED,
-  Event.VIEW_CALCULATION_UPDATED,
-  Event.AUTOMATION_TRIGGER_UPDATED,
-  Event.USER_GROUP_UPDATED,
-]
-
-export default class PosthogProcessor implements EventProcessor {
-  posthog: PostHog
-
-  constructor(token: string | undefined) {
-    if (!token) {
-      throw new Error("Posthog token is not defined")
-    }
-    this.posthog = new PostHog(token)
-  }
-
-  async processEvent(
-    event: Event,
-    identity: Identity,
-    properties: BaseEvent,
-    timestamp?: string | number
-  ): Promise<void> {
-    // don't send excluded events
-    if (EXCLUDED_EVENTS.includes(event)) {
-      return
-    }
-
-    if (await rateLimiting.limited(event)) {
-      return
-    }
-
-    properties = this.clearPIIProperties(properties)
-
-    properties.version = env.VERSION
-    properties.service = env.SERVICE
-    properties.environment = identity.environment
-    properties.hosting = identity.hosting
-
-    const appId = context.getAppId()
-    if (appId) {
-      properties.appId = appId
-    }
-
-    const payload: any = { distinctId: identity.id, event, properties }
-
-    if (timestamp) {
-      payload.timestamp = new Date(timestamp)
-    }
-
-    // add groups to the event
-    if (identity.installationId || identity.tenantId) {
-      payload.groups = {}
-      if (identity.installationId) {
-        payload.groups.installation = identity.installationId
-        payload.properties.installationId = identity.installationId
-      }
-      if (identity.tenantId) {
-        payload.groups.tenant = identity.tenantId
-        payload.properties.tenantId = identity.tenantId
-      }
-    }
-
-    this.posthog.capture(payload)
-  }
-
-  clearPIIProperties(properties: any) {
-    if (properties.email) {
-      delete properties.email
-    }
-    if (properties.audited) {
-      delete properties.audited
-    }
-    return properties
-  }
-
-  async identify(identity: Identity, timestamp?: string | number) {
-    const payload: any = { distinctId: identity.id, properties: identity }
-    if (timestamp) {
-      payload.timestamp = new Date(timestamp)
-    }
-    this.posthog.identify(payload)
-  }
-
-  async identifyGroup(group: Group, timestamp?: string | number) {
-    const payload: any = {
-      distinctId: group.id,
-      groupType: group.type,
-      groupKey: group.id,
-      properties: group,
-    }
-
-    if (timestamp) {
-      payload.timestamp = new Date(timestamp)
-    }
-    this.posthog.groupIdentify(payload)
-  }
-
-  shutdown() {
-    this.posthog.shutdown()
-  }
-}

package/src/events/processors/posthog/index.ts

@@ -1,2 +0,0 @@
-import PosthogProcessor from "./PosthogProcessor"
-export default PosthogProcessor

package/src/events/processors/posthog/rateLimiting.ts

@@ -1,106 +0,0 @@
-import { Event } from "@budibase/types"
-import { CacheKey, TTL } from "../../../cache/generic"
-import * as cache from "../../../cache/generic"
-import * as context from "../../../context"
-
-type RateLimitedEvent =
-  | Event.SERVED_BUILDER
-  | Event.SERVED_APP_PREVIEW
-  | Event.SERVED_APP
-
-const isRateLimited = (event: Event): event is RateLimitedEvent => {
-  return (
-    event === Event.SERVED_BUILDER ||
-    event === Event.SERVED_APP_PREVIEW ||
-    event === Event.SERVED_APP
-  )
-}
-
-const isPerApp = (event: RateLimitedEvent) => {
-  return event === Event.SERVED_APP_PREVIEW || event === Event.SERVED_APP
-}
-
-interface EventProperties {
-  timestamp: number
-}
-
-enum RateLimit {
-  CALENDAR_DAY = "calendarDay",
-}
-
-const RATE_LIMITS = {
-  [Event.SERVED_APP]: RateLimit.CALENDAR_DAY,
-  [Event.SERVED_APP_PREVIEW]: RateLimit.CALENDAR_DAY,
-  [Event.SERVED_BUILDER]: RateLimit.CALENDAR_DAY,
-}
-
-/**
- * Check if this event should be sent right now
- * Return false to signal the event SHOULD be sent
- * Return true to signal the event should NOT be sent
- */
-export const limited = async (event: Event): Promise<boolean> => {
-  // not a rate limited event -- send
-  if (!isRateLimited(event)) {
-    return false
-  }
-
-  const cachedEvent = await readEvent(event)
-  if (cachedEvent) {
-    const timestamp = new Date(cachedEvent.timestamp)
-    const limit = RATE_LIMITS[event]
-    switch (limit) {
-      case RateLimit.CALENDAR_DAY: {
-        // get midnight at the start of the next day for the timestamp
-        timestamp.setDate(timestamp.getDate() + 1)
-        timestamp.setHours(0, 0, 0, 0)
-
-        // if we have passed the threshold into the next day
-        if (Date.now() > timestamp.getTime()) {
-          // update the timestamp in the event -- send
-          await recordEvent(event, { timestamp: Date.now() })
-          return false
-        } else {
-          // still within the limited period -- don't send
-          return true
-        }
-      }
-    }
-  } else {
-    // no event present i.e. expired -- send
-    await recordEvent(event, { timestamp: Date.now() })
-    return false
-  }
-}
-
-const eventKey = (event: RateLimitedEvent) => {
-  let key = `${CacheKey.EVENTS_RATE_LIMIT}:${event}`
-  if (isPerApp(event)) {
-    key = key + ":" + context.getAppId()
-  }
-  return key
-}
-
-const readEvent = async (
-  event: RateLimitedEvent
-): Promise<EventProperties | undefined> => {
-  const key = eventKey(event)
-  const result = await cache.get(key)
-  return result as EventProperties
-}
-
-const recordEvent = async (
-  event: RateLimitedEvent,
-  properties: EventProperties
-) => {
-  const key = eventKey(event)
-  const limit = RATE_LIMITS[event]
-  let ttl
-  switch (limit) {
-    case RateLimit.CALENDAR_DAY: {
-      ttl = TTL.ONE_DAY
-    }
-  }
-
-  await cache.store(key, properties, ttl)
-}

package/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,168 +0,0 @@
-import { testEnv } from "../../../../../tests/extra"
-import PosthogProcessor from "../PosthogProcessor"
-import { Event, IdentityType, Hosting } from "@budibase/types"
-const tk = require("timekeeper")
-import * as cache from "../../../../cache/generic"
-import { CacheKey } from "../../../../cache/generic"
-import * as context from "../../../../context"
-
-const newIdentity = () => {
-  return {
-    id: "test",
-    type: IdentityType.USER,
-    hosting: Hosting.SELF,
-    environment: "test",
-  }
-}
-
-describe("PosthogProcessor", () => {
-  beforeAll(() => {
-    testEnv.singleTenant()
-  })
-
-  beforeEach(async () => {
-    jest.clearAllMocks()
-    await cache.bustCache(
-      `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
-    )
-  })
-
-  describe("processEvent", () => {
-    it("processes event", async () => {
-      const processor = new PosthogProcessor("test")
-
-      const identity = newIdentity()
-      const properties = {}
-
-      await processor.processEvent(Event.APP_CREATED, identity, properties)
-
-      expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
-    })
-
-    it("honours exclusions", async () => {
-      const processor = new PosthogProcessor("test")
-
-      const identity = newIdentity()
-      const properties = {}
-
-      await processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties)
-      expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
-    })
-
-    it("removes audited information", async () => {
-      const processor = new PosthogProcessor("test")
-
-      const identity = newIdentity()
-      const properties = {
-        email: "test",
-        audited: {
-          name: "test",
-        },
-      }
-
-      await processor.processEvent(Event.USER_CREATED, identity, properties)
-      expect(processor.posthog.capture).toHaveBeenCalled()
-      // @ts-ignore
-      const call = processor.posthog.capture.mock.calls[0][0]
-      expect(call.properties.audited).toBeUndefined()
-      expect(call.properties.email).toBeUndefined()
-    })
-
-    describe("rate limiting", () => {
-      it("sends daily event once in same day", async () => {
-        const processor = new PosthogProcessor("test")
-        const identity = newIdentity()
-        const properties = {}
-
-        tk.freeze(new Date(2022, 0, 1, 14, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-        // go forward one hour
-        tk.freeze(new Date(2022, 0, 1, 15, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
-      })
-
-      it("sends daily event once per unique day", async () => {
-        const processor = new PosthogProcessor("test")
-        const identity = newIdentity()
-        const properties = {}
-
-        tk.freeze(new Date(2022, 0, 1, 14, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-        // go forward into next day
-        tk.freeze(new Date(2022, 0, 2, 9, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-        // go forward into next day
-        tk.freeze(new Date(2022, 0, 3, 5, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-        // go forward one hour
-        tk.freeze(new Date(2022, 0, 3, 6, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(3)
-      })
-
-      it("sends event again after cache expires", async () => {
-        const processor = new PosthogProcessor("test")
-        const identity = newIdentity()
-        const properties = {}
-
-        tk.freeze(new Date(2022, 0, 1, 14, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-
-        await cache.bustCache(
-          `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
-        )
-
-        tk.freeze(new Date(2022, 0, 1, 14, 0))
-        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
-
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(2)
-      })
-
-      it("sends per app events once per day per app", async () => {
-        const processor = new PosthogProcessor("test")
-        const identity = newIdentity()
-        const properties = {}
-
-        const runAppEvents = async (appId: string) => {
-          await context.doInAppContext(appId, async () => {
-            tk.freeze(new Date(2022, 0, 1, 14, 0))
-            await processor.processEvent(Event.SERVED_APP, identity, properties)
-            await processor.processEvent(
-              Event.SERVED_APP_PREVIEW,
-              identity,
-              properties
-            )
-
-            // go forward one hour - should be ignored
-            tk.freeze(new Date(2022, 0, 1, 15, 0))
-            await processor.processEvent(Event.SERVED_APP, identity, properties)
-            await processor.processEvent(
-              Event.SERVED_APP_PREVIEW,
-              identity,
-              properties
-            )
-
-            // go forward into next day
-            tk.freeze(new Date(2022, 0, 2, 9, 0))
-
-            await processor.processEvent(Event.SERVED_APP, identity, properties)
-            await processor.processEvent(
-              Event.SERVED_APP_PREVIEW,
-              identity,
-              properties
-            )
-          })
-        }
-
-        await runAppEvents("app_1")
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(4)
-
-        await runAppEvents("app_2")
-        expect(processor.posthog.capture).toHaveBeenCalledTimes(8)
-      })
-    })
-  })
-})

package/src/events/processors/types.ts

@@ -1 +0,0 @@
-export { EventProcessor } from "@budibase/types"

package/src/events/publishers/account.ts

@@ -1,35 +0,0 @@
-import { publishEvent } from "../events"
-import {
-  Event,
-  Account,
-  AccountCreatedEvent,
-  AccountDeletedEvent,
-  AccountVerifiedEvent,
-} from "@budibase/types"
-
-async function created(account: Account) {
-  const properties: AccountCreatedEvent = {
-    tenantId: account.tenantId,
-  }
-  await publishEvent(Event.ACCOUNT_CREATED, properties)
-}
-
-async function deleted(account: Account) {
-  const properties: AccountDeletedEvent = {
-    tenantId: account.tenantId,
-  }
-  await publishEvent(Event.ACCOUNT_DELETED, properties)
-}
-
-async function verified(account: Account) {
-  const properties: AccountVerifiedEvent = {
-    tenantId: account.tenantId,
-  }
-  await publishEvent(Event.ACCOUNT_VERIFIED, properties)
-}
-
-export default {
-  created,
-  deleted,
-  verified,
-}