@budibase/backend-core 2.9.40-alpha.6 → 2.10.1

package/src/events/processors/AuditLogsProcessor.ts

@@ -0,0 +1,93 @@
+import {
+  Event,
+  Identity,
+  Group,
+  IdentityType,
+  AuditLogQueueEvent,
+  AuditLogFn,
+  HostInfo,
+} from "@budibase/types"
+import { EventProcessor } from "./types"
+import { getAppId, doInTenant, getTenantId } from "../../context"
+import BullQueue from "bull"
+import { createQueue, JobQueue } from "../../queue"
+import { isAudited } from "../../utils"
+import env from "../../environment"
+
+export default class AuditLogsProcessor implements EventProcessor {
+  static auditLogsEnabled = false
+  static auditLogQueue: BullQueue.Queue<AuditLogQueueEvent>
+
+  // can't use constructor as need to return promise
+  static init(fn: AuditLogFn) {
+    AuditLogsProcessor.auditLogsEnabled = true
+    const writeAuditLogs = fn
+    AuditLogsProcessor.auditLogQueue = createQueue<AuditLogQueueEvent>(
+      JobQueue.AUDIT_LOG
+    )
+    return AuditLogsProcessor.auditLogQueue.process(async job => {
+      return doInTenant(job.data.tenantId, async () => {
+        let properties = job.data.properties
+        if (properties.audited) {
+          properties = {
+            ...properties,
+            ...properties.audited,
+          }
+          delete properties.audited
+        }
+
+        // this feature is disabled by default due to privacy requirements
+        // in some countries - available as env var in-case it is desired
+        // in self host deployments
+        let hostInfo: HostInfo | undefined = {}
+        if (env.ENABLE_AUDIT_LOG_IP_ADDR) {
+          hostInfo = job.data.opts.hostInfo
+        }
+
+        await writeAuditLogs(job.data.event, properties, {
+          userId: job.data.opts.userId,
+          timestamp: job.data.opts.timestamp,
+          appId: job.data.opts.appId,
+          hostInfo,
+        })
+      })
+    })
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (AuditLogsProcessor.auditLogsEnabled && isAudited(event)) {
+      // only audit log actual events, don't include backfills
+      const userId =
+        identity.type === IdentityType.USER ? identity.id : undefined
+      // add to the event queue, rather than just writing immediately
+      await AuditLogsProcessor.auditLogQueue.add({
+        event,
+        properties,
+        opts: {
+          userId,
+          timestamp,
+          appId: getAppId(),
+          hostInfo: identity.hostInfo,
+        },
+        tenantId: getTenantId(),
+      })
+    }
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    // no-op
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // no-op
+  }
+
+  shutdown(): void {
+    AuditLogsProcessor.auditLogQueue?.close()
+  }
+}

package/src/events/processors/LoggingProcessor.ts

@@ -0,0 +1,37 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+
+const skipLogging = env.SELF_HOSTED && !env.isDev()
+
+export default class LoggingProcessor implements EventProcessor {
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (skipLogging) {
+      return
+    }
+    console.log(`[audit] [identityType=${identity.type}] ${event}`, properties)
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    console.log(`[audit] identified`, identity)
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    console.log(`[audit] group identified`, group)
+  }
+
+  shutdown(): void {
+    // no-op
+  }
+}

package/src/events/processors/Processors.ts

@@ -0,0 +1,52 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+
+export default class Processor implements EventProcessor {
+  initialised: boolean = false
+  processors: EventProcessor[] = []
+
+  constructor(processors: EventProcessor[]) {
+    this.processors = processors
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(
+    identity: Identity,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      if (eventProcessor.identify) {
+        await eventProcessor.identify(identity, timestamp)
+      }
+    }
+  }
+
+  async identifyGroup(
+    identity: Group,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      if (eventProcessor.identifyGroup) {
+        await eventProcessor.identifyGroup(identity, timestamp)
+      }
+    }
+  }
+
+  shutdown() {
+    for (const eventProcessor of this.processors) {
+      if (eventProcessor.shutdown) {
+        eventProcessor.shutdown()
+      }
+    }
+  }
+}

package/src/events/processors/async/DocumentUpdateProcessor.ts

@@ -0,0 +1,43 @@
+import { EventProcessor } from "../types"
+import { Event, Identity, DocUpdateEvent } from "@budibase/types"
+import { doInTenant } from "../../../context"
+import { getDocumentId } from "../../documentId"
+import { shutdown } from "../../asyncEvents"
+
+export type Processor = (update: DocUpdateEvent) => Promise<void>
+export type ProcessorMap = { events: Event[]; processor: Processor }[]
+
+export default class DocumentUpdateProcessor implements EventProcessor {
+  processors: ProcessorMap = []
+
+  constructor(processors: ProcessorMap) {
+    this.processors = processors
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ) {
+    const tenantId = identity.realTenantId
+    const docId = getDocumentId(event, properties)
+    if (!tenantId || !docId) {
+      return
+    }
+    for (let { events, processor } of this.processors) {
+      if (events.includes(event)) {
+        await doInTenant(tenantId, async () => {
+          await processor({
+            id: docId,
+            tenantId,
+          })
+        })
+      }
+    }
+  }
+
+  shutdown() {
+    return shutdown()
+  }
+}

package/src/events/processors/index.ts

@@ -0,0 +1,19 @@
+import AnalyticsProcessor from "./AnalyticsProcessor"
+import LoggingProcessor from "./LoggingProcessor"
+import AuditLogsProcessor from "./AuditLogsProcessor"
+import Processors from "./Processors"
+import { AuditLogFn } from "@budibase/types"
+
+export const analyticsProcessor = new AnalyticsProcessor()
+const loggingProcessor = new LoggingProcessor()
+const auditLogsProcessor = new AuditLogsProcessor()
+
+export function init(auditingFn: AuditLogFn) {
+  return AuditLogsProcessor.init(auditingFn)
+}
+
+export const processors = new Processors([
+  analyticsProcessor,
+  loggingProcessor,
+  auditLogsProcessor,
+])

package/src/events/processors/posthog/PosthogProcessor.ts

@@ -0,0 +1,118 @@
+import PostHog from "posthog-node"
+import { Event, Identity, Group, BaseEvent } from "@budibase/types"
+import { EventProcessor } from "../types"
+import env from "../../../environment"
+import * as context from "../../../context"
+import * as rateLimiting from "./rateLimiting"
+
+const EXCLUDED_EVENTS: Event[] = [
+  Event.USER_UPDATED,
+  Event.EMAIL_SMTP_UPDATED,
+  Event.AUTH_SSO_UPDATED,
+  Event.APP_UPDATED,
+  Event.ROLE_UPDATED,
+  Event.DATASOURCE_UPDATED,
+  Event.QUERY_UPDATED,
+  Event.TABLE_UPDATED,
+  Event.VIEW_UPDATED,
+  Event.VIEW_FILTER_UPDATED,
+  Event.VIEW_CALCULATION_UPDATED,
+  Event.AUTOMATION_TRIGGER_UPDATED,
+  Event.USER_GROUP_UPDATED,
+]
+
+export default class PosthogProcessor implements EventProcessor {
+  posthog: PostHog
+
+  constructor(token: string | undefined) {
+    if (!token) {
+      throw new Error("Posthog token is not defined")
+    }
+    this.posthog = new PostHog(token)
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: BaseEvent,
+    timestamp?: string | number
+  ): Promise<void> {
+    // don't send excluded events
+    if (EXCLUDED_EVENTS.includes(event)) {
+      return
+    }
+
+    if (await rateLimiting.limited(event)) {
+      return
+    }
+
+    properties = this.clearPIIProperties(properties)
+
+    properties.version = env.VERSION
+    properties.service = env.SERVICE
+    properties.environment = identity.environment
+    properties.hosting = identity.hosting
+
+    const appId = context.getAppId()
+    if (appId) {
+      properties.appId = appId
+    }
+
+    const payload: any = { distinctId: identity.id, event, properties }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+
+    // add groups to the event
+    if (identity.installationId || identity.tenantId) {
+      payload.groups = {}
+      if (identity.installationId) {
+        payload.groups.installation = identity.installationId
+        payload.properties.installationId = identity.installationId
+      }
+      if (identity.tenantId) {
+        payload.groups.tenant = identity.tenantId
+        payload.properties.tenantId = identity.tenantId
+      }
+    }
+
+    this.posthog.capture(payload)
+  }
+
+  clearPIIProperties(properties: any) {
+    if (properties.email) {
+      delete properties.email
+    }
+    if (properties.audited) {
+      delete properties.audited
+    }
+    return properties
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    const payload: any = { distinctId: identity.id, properties: identity }
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.identify(payload)
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    const payload: any = {
+      distinctId: group.id,
+      groupType: group.type,
+      groupKey: group.id,
+      properties: group,
+    }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.groupIdentify(payload)
+  }
+
+  shutdown() {
+    this.posthog.shutdown()
+  }
+}

package/src/events/processors/posthog/index.ts

@@ -0,0 +1,2 @@
+import PosthogProcessor from "./PosthogProcessor"
+export default PosthogProcessor

package/src/events/processors/posthog/rateLimiting.ts

@@ -0,0 +1,106 @@
+import { Event } from "@budibase/types"
+import { CacheKey, TTL } from "../../../cache/generic"
+import * as cache from "../../../cache/generic"
+import * as context from "../../../context"
+
+type RateLimitedEvent =
+  | Event.SERVED_BUILDER
+  | Event.SERVED_APP_PREVIEW
+  | Event.SERVED_APP
+
+const isRateLimited = (event: Event): event is RateLimitedEvent => {
+  return (
+    event === Event.SERVED_BUILDER ||
+    event === Event.SERVED_APP_PREVIEW ||
+    event === Event.SERVED_APP
+  )
+}
+
+const isPerApp = (event: RateLimitedEvent) => {
+  return event === Event.SERVED_APP_PREVIEW || event === Event.SERVED_APP
+}
+
+interface EventProperties {
+  timestamp: number
+}
+
+enum RateLimit {
+  CALENDAR_DAY = "calendarDay",
+}
+
+const RATE_LIMITS = {
+  [Event.SERVED_APP]: RateLimit.CALENDAR_DAY,
+  [Event.SERVED_APP_PREVIEW]: RateLimit.CALENDAR_DAY,
+  [Event.SERVED_BUILDER]: RateLimit.CALENDAR_DAY,
+}
+
+/**
+ * Check if this event should be sent right now
+ * Return false to signal the event SHOULD be sent
+ * Return true to signal the event should NOT be sent
+ */
+export const limited = async (event: Event): Promise<boolean> => {
+  // not a rate limited event -- send
+  if (!isRateLimited(event)) {
+    return false
+  }
+
+  const cachedEvent = await readEvent(event)
+  if (cachedEvent) {
+    const timestamp = new Date(cachedEvent.timestamp)
+    const limit = RATE_LIMITS[event]
+    switch (limit) {
+      case RateLimit.CALENDAR_DAY: {
+        // get midnight at the start of the next day for the timestamp
+        timestamp.setDate(timestamp.getDate() + 1)
+        timestamp.setHours(0, 0, 0, 0)
+
+        // if we have passed the threshold into the next day
+        if (Date.now() > timestamp.getTime()) {
+          // update the timestamp in the event -- send
+          await recordEvent(event, { timestamp: Date.now() })
+          return false
+        } else {
+          // still within the limited period -- don't send
+          return true
+        }
+      }
+    }
+  } else {
+    // no event present i.e. expired -- send
+    await recordEvent(event, { timestamp: Date.now() })
+    return false
+  }
+}
+
+const eventKey = (event: RateLimitedEvent) => {
+  let key = `${CacheKey.EVENTS_RATE_LIMIT}:${event}`
+  if (isPerApp(event)) {
+    key = key + ":" + context.getAppId()
+  }
+  return key
+}
+
+const readEvent = async (
+  event: RateLimitedEvent
+): Promise<EventProperties | undefined> => {
+  const key = eventKey(event)
+  const result = await cache.get(key)
+  return result as EventProperties
+}
+
+const recordEvent = async (
+  event: RateLimitedEvent,
+  properties: EventProperties
+) => {
+  const key = eventKey(event)
+  const limit = RATE_LIMITS[event]
+  let ttl
+  switch (limit) {
+    case RateLimit.CALENDAR_DAY: {
+      ttl = TTL.ONE_DAY
+    }
+  }
+
+  await cache.store(key, properties, ttl)
+}

package/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -0,0 +1,168 @@
+import { testEnv } from "../../../../../tests/extra"
+import PosthogProcessor from "../PosthogProcessor"
+import { Event, IdentityType, Hosting } from "@budibase/types"
+const tk = require("timekeeper")
+import * as cache from "../../../../cache/generic"
+import { CacheKey } from "../../../../cache/generic"
+import * as context from "../../../../context"
+
+const newIdentity = () => {
+  return {
+    id: "test",
+    type: IdentityType.USER,
+    hosting: Hosting.SELF,
+    environment: "test",
+  }
+}
+
+describe("PosthogProcessor", () => {
+  beforeAll(() => {
+    testEnv.singleTenant()
+  })
+
+  beforeEach(async () => {
+    jest.clearAllMocks()
+    await cache.bustCache(
+      `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
+    )
+  })
+
+  describe("processEvent", () => {
+    it("processes event", async () => {
+      const processor = new PosthogProcessor("test")
+
+      const identity = newIdentity()
+      const properties = {}
+
+      await processor.processEvent(Event.APP_CREATED, identity, properties)
+
+      expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
+    })
+
+    it("honours exclusions", async () => {
+      const processor = new PosthogProcessor("test")
+
+      const identity = newIdentity()
+      const properties = {}
+
+      await processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties)
+      expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
+    })
+
+    it("removes audited information", async () => {
+      const processor = new PosthogProcessor("test")
+
+      const identity = newIdentity()
+      const properties = {
+        email: "test",
+        audited: {
+          name: "test",
+        },
+      }
+
+      await processor.processEvent(Event.USER_CREATED, identity, properties)
+      expect(processor.posthog.capture).toHaveBeenCalled()
+      // @ts-ignore
+      const call = processor.posthog.capture.mock.calls[0][0]
+      expect(call.properties.audited).toBeUndefined()
+      expect(call.properties.email).toBeUndefined()
+    })
+
+    describe("rate limiting", () => {
+      it("sends daily event once in same day", async () => {
+        const processor = new PosthogProcessor("test")
+        const identity = newIdentity()
+        const properties = {}
+
+        tk.freeze(new Date(2022, 0, 1, 14, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+        // go forward one hour
+        tk.freeze(new Date(2022, 0, 1, 15, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+        expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
+      })
+
+      it("sends daily event once per unique day", async () => {
+        const processor = new PosthogProcessor("test")
+        const identity = newIdentity()
+        const properties = {}
+
+        tk.freeze(new Date(2022, 0, 1, 14, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+        // go forward into next day
+        tk.freeze(new Date(2022, 0, 2, 9, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+        // go forward into next day
+        tk.freeze(new Date(2022, 0, 3, 5, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+        // go forward one hour
+        tk.freeze(new Date(2022, 0, 3, 6, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+        expect(processor.posthog.capture).toHaveBeenCalledTimes(3)
+      })
+
+      it("sends event again after cache expires", async () => {
+        const processor = new PosthogProcessor("test")
+        const identity = newIdentity()
+        const properties = {}
+
+        tk.freeze(new Date(2022, 0, 1, 14, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+        await cache.bustCache(
+          `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
+        )
+
+        tk.freeze(new Date(2022, 0, 1, 14, 0))
+        await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+        expect(processor.posthog.capture).toHaveBeenCalledTimes(2)
+      })
+
+      it("sends per app events once per day per app", async () => {
+        const processor = new PosthogProcessor("test")
+        const identity = newIdentity()
+        const properties = {}
+
+        const runAppEvents = async (appId: string) => {
+          await context.doInAppContext(appId, async () => {
+            tk.freeze(new Date(2022, 0, 1, 14, 0))
+            await processor.processEvent(Event.SERVED_APP, identity, properties)
+            await processor.processEvent(
+              Event.SERVED_APP_PREVIEW,
+              identity,
+              properties
+            )
+
+            // go forward one hour - should be ignored
+            tk.freeze(new Date(2022, 0, 1, 15, 0))
+            await processor.processEvent(Event.SERVED_APP, identity, properties)
+            await processor.processEvent(
+              Event.SERVED_APP_PREVIEW,
+              identity,
+              properties
+            )
+
+            // go forward into next day
+            tk.freeze(new Date(2022, 0, 2, 9, 0))
+
+            await processor.processEvent(Event.SERVED_APP, identity, properties)
+            await processor.processEvent(
+              Event.SERVED_APP_PREVIEW,
+              identity,
+              properties
+            )
+          })
+        }
+
+        await runAppEvents("app_1")
+        expect(processor.posthog.capture).toHaveBeenCalledTimes(4)
+
+        await runAppEvents("app_2")
+        expect(processor.posthog.capture).toHaveBeenCalledTimes(8)
+      })
+    })
+  })
+})

package/src/events/processors/types.ts

@@ -0,0 +1 @@
+export { EventProcessor } from "@budibase/types"

package/src/events/publishers/account.ts

@@ -0,0 +1,35 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Account,
+  AccountCreatedEvent,
+  AccountDeletedEvent,
+  AccountVerifiedEvent,
+} from "@budibase/types"
+
+async function created(account: Account) {
+  const properties: AccountCreatedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_CREATED, properties)
+}
+
+async function deleted(account: Account) {
+  const properties: AccountDeletedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_DELETED, properties)
+}
+
+async function verified(account: Account) {
+  const properties: AccountVerifiedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_VERIFIED, properties)
+}
+
+export default {
+  created,
+  deleted,
+  verified,
+}