@budibase/backend-core 2.30.2 → 2.30.4

package/src/features/index.ts

@@ -1,108 +1,270 @@
 import env from "../environment"
 import * as context from "../context"
-import { cloneDeep } from "lodash"
+import { PostHog, PostHogOptions } from "posthog-node"
+import { IdentityType, UserCtx } from "@budibase/types"
+import tracer from "dd-trace"
 
-class Flag<T> {
-  static withDefault<T>(value: T) {
-    return new Flag(value)
+let posthog: PostHog | undefined
+export function init(opts?: PostHogOptions) {
+  if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST) {
+    console.log("initializing posthog client...")
+    posthog = new PostHog(env.POSTHOG_TOKEN, {
+      host: env.POSTHOG_API_HOST,
+      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
+      ...opts,
+    })
+  } else {
+    console.log("posthog disabled")
   }
-
-  private constructor(public defaultValue: T) {}
 }
 
-// This is the primary source of truth for feature flags. If you want to add a
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-const FLAGS = {
-  LICENSING: Flag.withDefault(false),
-  GOOGLE_SHEETS: Flag.withDefault(false),
-  USER_GROUPS: Flag.withDefault(false),
-  ONBOARDING_TOUR: Flag.withDefault(false),
+export function shutdown() {
+  posthog?.shutdown()
 }
 
-const DEFAULTS = Object.keys(FLAGS).reduce((acc, key) => {
-  const typedKey = key as keyof typeof FLAGS
-  // @ts-ignore
-  acc[typedKey] = FLAGS[typedKey].defaultValue
-  return acc
-}, {} as Flags)
+export abstract class Flag<T> {
+  static boolean(defaultValue: boolean): Flag<boolean> {
+    return new BooleanFlag(defaultValue)
+  }
+
+  static string(defaultValue: string): Flag<string> {
+    return new StringFlag(defaultValue)
+  }
+
+  static number(defaultValue: number): Flag<number> {
+    return new NumberFlag(defaultValue)
+  }
+
+  protected constructor(public defaultValue: T) {}
+
+  abstract parse(value: any): T
+}
 
 type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-export type Flags = {
-  [K in keyof typeof FLAGS]: UnwrapFlag<(typeof FLAGS)[K]>
+
+export type FlagValues<T> = {
+  [K in keyof T]: UnwrapFlag<T[K]>
 }
 
-// Exported for use in tests, should not be used outside of this file.
-export function defaultFlags(): Flags {
-  return cloneDeep(DEFAULTS)
+type KeysOfType<T, U> = {
+  [K in keyof T]: T[K] extends Flag<U> ? K : never
+}[keyof T]
+
+class BooleanFlag extends Flag<boolean> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return ["true", "t", "1"].includes(value.toLowerCase())
+    }
+
+    if (typeof value === "boolean") {
+      return value
+    }
+
+    throw new Error(`could not parse value "${value}" as boolean`)
+  }
 }
 
-function isFlagName(name: string): name is keyof Flags {
-  return FLAGS[name as keyof typeof FLAGS] !== undefined
+class StringFlag extends Flag<string> {
+  parse(value: any) {
+    if (typeof value === "string") {
+      return value
+    }
+    throw new Error(`could not parse value "${value}" as string`)
+  }
 }
 
-/**
- * Reads the TENANT_FEATURE_FLAGS environment variable and returns a Flags object
- * populated with the flags for the current tenant, filling in the default values
- * if the flag is not set.
- *
- * Check the tests for examples of how TENANT_FEATURE_FLAGS should be formatted.
- *
- * In future we plan to add more ways of setting feature flags, e.g. PostHog, and
- * they will be accessed through this function as well.
- */
-export async function fetch(): Promise<Flags> {
-  const currentTenantId = context.getTenantId()
-  const flags = defaultFlags()
-
-  const split = (env.TENANT_FEATURE_FLAGS || "")
-    .split(",")
-    .map(x => x.split(":"))
-  for (const [tenantId, ...features] of split) {
-    if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
-      continue
+class NumberFlag extends Flag<number> {
+  parse(value: any) {
+    if (typeof value === "number") {
+      return value
     }
 
-    for (let feature of features) {
-      let value = true
-      if (feature.startsWith("!")) {
-        feature = feature.slice(1)
-        value = false
+    if (typeof value === "string") {
+      const parsed = parseFloat(value)
+      if (!isNaN(parsed)) {
+        return parsed
       }
+    }
+
+    throw new Error(`could not parse value "${value}" as number`)
+  }
+}
+
+export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
+  // This is used to safely cache flags sets in the current request context.
+  // Because multiple sets could theoretically exist, we don't want the cache of
+  // one to leak into another.
+  private readonly setId: string
+
+  constructor(private readonly flagSchema: T) {
+    this.setId = crypto.randomUUID()
+  }
+
+  defaults(): FlagValues<T> {
+    return Object.keys(this.flagSchema).reduce((acc, key) => {
+      const typedKey = key as keyof T
+      acc[typedKey] = this.flagSchema[key].defaultValue
+      return acc
+    }, {} as FlagValues<T>)
+  }
+
+  isFlagName(name: string | number | symbol): name is keyof T {
+    return this.flagSchema[name as keyof T] !== undefined
+  }
 
-      if (!isFlagName(feature)) {
-        throw new Error(`Feature: ${feature} is not an allowed option`)
+  async get<K extends keyof T>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<FlagValues<T>[K]> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async isEnabled<K extends KeysOfType<T, boolean>>(
+    key: K,
+    ctx?: UserCtx
+  ): Promise<boolean> {
+    const flags = await this.fetch(ctx)
+    return flags[key]
+  }
+
+  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
+    return await tracer.trace("features.fetch", async span => {
+      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
+      if (cachedFlags) {
+        span?.addTags({ fromCache: true })
+        return cachedFlags
       }
 
-      if (typeof flags[feature] !== "boolean") {
-        throw new Error(`Feature: ${feature} is not a boolean`)
+      const tags: Record<string, any> = {}
+      const flagValues = this.defaults()
+      const currentTenantId = context.getTenantId()
+      const specificallySetFalse = new Set<string>()
+
+      const split = (env.TENANT_FEATURE_FLAGS || "")
+        .split(",")
+        .map(x => x.split(":"))
+      for (const [tenantId, ...features] of split) {
+        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
+          continue
+        }
+
+        tags[`readFromEnvironmentVars`] = true
+
+        for (let feature of features) {
+          let value = true
+          if (feature.startsWith("!")) {
+            feature = feature.slice(1)
+            value = false
+            specificallySetFalse.add(feature)
+          }
+
+          // ignore unknown flags
+          if (!this.isFlagName(feature)) {
+            continue
+          }
+
+          if (typeof flagValues[feature] !== "boolean") {
+            throw new Error(`Feature: ${feature} is not a boolean`)
+          }
+
+          // @ts-expect-error - TS does not like you writing into a generic type,
+          // but we know that it's okay in this case because it's just an object.
+          flagValues[feature as keyof FlagValues] = value
+          tags[`flags.${feature}.source`] = "environment"
+        }
       }
 
-      // @ts-ignore
-      flags[feature] = value
-    }
-  }
+      const license = ctx?.user?.license
+      if (license) {
+        tags[`readFromLicense`] = true
 
-  return flags
-}
+        for (const feature of license.features) {
+          if (!this.isFlagName(feature)) {
+            continue
+          }
 
-// Gets a single feature flag value. This is a convenience function for
-// `fetch().then(flags => flags[name])`.
-export async function get<K extends keyof Flags>(name: K): Promise<Flags[K]> {
-  const flags = await fetch()
-  return flags[name]
-}
+          if (
+            flagValues[feature] === true ||
+            specificallySetFalse.has(feature)
+          ) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          // @ts-expect-error - TS does not like you writing into a generic type,
+          // but we know that it's okay in this case because it's just an object.
+          flagValues[feature] = true
+          tags[`flags.${feature}.source`] = "license"
+        }
+      }
 
-type BooleanFlags = {
-  [K in keyof typeof FLAGS]: (typeof FLAGS)[K] extends Flag<boolean> ? K : never
-}[keyof typeof FLAGS]
-
-// Convenience function for boolean flag values. This makes callsites more
-// readable for boolean flags.
-export async function isEnabled<K extends BooleanFlags>(
-  name: K
-): Promise<boolean> {
-  const flags = await fetch()
-  return flags[name]
+      const identity = context.getIdentity()
+      tags[`identity.type`] = identity?.type
+      tags[`identity.tenantId`] = identity?.tenantId
+      tags[`identity._id`] = identity?._id
+
+      if (posthog && identity?.type === IdentityType.USER) {
+        tags[`readFromPostHog`] = true
+
+        const personProperties: Record<string, string> = {}
+        if (identity.tenantId) {
+          personProperties.tenantId = identity.tenantId
+        }
+
+        const posthogFlags = await posthog.getAllFlagsAndPayloads(
+          identity._id,
+          {
+            personProperties,
+          }
+        )
+
+        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
+          if (!this.isFlagName(name)) {
+            // We don't want an unexpected PostHog flag to break the app, so we
+            // just log it and continue.
+            console.warn(`Unexpected posthog flag "${name}": ${value}`)
+            continue
+          }
+
+          if (flagValues[name] === true || specificallySetFalse.has(name)) {
+            // If the flag is already set to through environment variables, we
+            // don't want to override it back to false here.
+            continue
+          }
+
+          const payload = posthogFlags.featureFlagPayloads?.[name]
+          const flag = this.flagSchema[name]
+          try {
+            // @ts-expect-error - TS does not like you writing into a generic
+            // type, but we know that it's okay in this case because it's just
+            // an object.
+            flagValues[name] = flag.parse(payload || value)
+            tags[`flags.${name}.source`] = "posthog"
+          } catch (err) {
+            // We don't want an invalid PostHog flag to break the app, so we just
+            // log it and continue.
+            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
+          }
+        }
+      }
+
+      context.setFeatureFlags(this.setId, flagValues)
+      for (const [key, value] of Object.entries(flagValues)) {
+        tags[`flags.${key}.value`] = value
+      }
+      span?.addTags(tags)
+
+      return flagValues
+    })
+  }
 }
+
+// This is the primary source of truth for feature flags. If you want to add a
+// new flag, add it here and use the `fetch` and `get` functions to access it.
+// All of the machinery in this file is to make sure that flags have their
+// default values set correctly and their types flow through the system.
+export const flags = new FlagSet({
+  DEFAULT_VALUES: Flag.boolean(false),
+})

package/src/features/tests/features.spec.ts

@@ -1,86 +1,232 @@
-import { defaultFlags, fetch, get, Flags } from "../"
-import { context } from "../.."
-import env from "../../environment"
-
-async function withFlags<T>(flags: string, f: () => T): Promise<T> {
-  const oldFlags = env.TENANT_FEATURE_FLAGS
-  env._set("TENANT_FEATURE_FLAGS", flags)
-  try {
-    return await f()
-  } finally {
-    env._set("TENANT_FEATURE_FLAGS", oldFlags)
-  }
+import { IdentityContext, IdentityType, UserCtx } from "@budibase/types"
+import { Flag, FlagSet, FlagValues, init, shutdown } from "../"
+import * as context from "../../context"
+import environment, { withEnv } from "../../environment"
+import nodeFetch from "node-fetch"
+import nock from "nock"
+
+const schema = {
+  TEST_BOOLEAN: Flag.boolean(false),
+  TEST_STRING: Flag.string("default value"),
+  TEST_NUMBER: Flag.number(0),
+}
+const flags = new FlagSet(schema)
+
+interface TestCase {
+  it: string
+  identity?: Partial<IdentityContext>
+  environmentFlags?: string
+  posthogFlags?: PostHogFlags
+  licenseFlags?: Array<string>
+  expected?: Partial<FlagValues<typeof schema>>
+  errorMessage?: string | RegExp
+}
+
+interface PostHogFlags {
+  featureFlags?: Record<string, boolean>
+  featureFlagPayloads?: Record<string, string>
+}
+
+function mockPosthogFlags(flags: PostHogFlags) {
+  nock("https://us.i.posthog.com")
+    .post("/decide/?v=3", body => {
+      return body.token === "test" && body.distinct_id === "us_1234"
+    })
+    .reply(200, flags)
+    .persist()
 }
 
 describe("feature flags", () => {
-  interface TestCase {
-    tenant: string
-    flags: string
-    expected: Partial<Flags>
-  }
+  beforeEach(() => {
+    nock.cleanAll()
+  })
 
   it.each<TestCase>([
     {
-      tenant: "tenant1",
-      flags: "tenant1:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: true },
+      it: "should should find a simple boolean flag in the environment",
+      environmentFlags: "default:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should should find a simple netgative boolean flag in the environment",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should should match stars in the environment",
+      environmentFlags: "*:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: true },
     },
     {
-      tenant: "tenant1",
-      flags: "tenant1:!ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: false },
+      it: "should not match a different tenant's flags",
+      environmentFlags: "otherTenant:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: false },
     },
     {
-      tenant: "tenant1",
-      flags: "*:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: true },
+      it: "should return the defaults when no flags are set",
+      expected: flags.defaults(),
     },
     {
-      tenant: "tenant1",
-      flags: "tenant2:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: false },
+      it: "should ignore unknown feature flags",
+      environmentFlags: "default:TEST_BOOLEAN,default:FOO",
+      expected: { TEST_BOOLEAN: true },
     },
     {
-      tenant: "tenant1",
-      flags: "",
-      expected: defaultFlags(),
+      it: "should be able to read boolean flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_BOOLEAN: true },
+      },
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should be able to read string flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_STRING: true },
+        featureFlagPayloads: { TEST_STRING: "test" },
+      },
+      expected: { TEST_STRING: "test" },
+    },
+    {
+      it: "should be able to read numeric flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_NUMBER: true },
+        featureFlagPayloads: { TEST_NUMBER: "123" },
+      },
+      expected: { TEST_NUMBER: 123 },
+    },
+    {
+      it: "should not be able to override a negative environment flag from PostHog",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      posthogFlags: {
+        featureFlags: { TEST_BOOLEAN: true },
+      },
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should not be able to override a positive environment flag from PostHog",
+      environmentFlags: "default:TEST_BOOLEAN",
+      posthogFlags: {
+        featureFlags: {
+          TEST_BOOLEAN: false,
+        },
+      },
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should be able to set boolean flags through the license",
+      licenseFlags: ["TEST_BOOLEAN"],
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should not be able to override a negative environment flag from license",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      licenseFlags: ["TEST_BOOLEAN"],
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should not error on unrecognised PostHog flag",
+      posthogFlags: {
+        featureFlags: { UNDEFINED: true },
+      },
+      expected: flags.defaults(),
+    },
+    {
+      it: "should not error on unrecognised license flag",
+      licenseFlags: ["UNDEFINED"],
+      expected: flags.defaults(),
     },
   ])(
-    'should find flags $expected for $tenant with string "$flags"',
-    ({ tenant, flags, expected }) =>
-      context.doInTenant(tenant, () =>
-        withFlags(flags, async () => {
-          const flags = await fetch()
-          expect(flags).toMatchObject(expected)
-
-          for (const [key, expectedValue] of Object.entries(expected)) {
-            const value = await get(key as keyof Flags)
-            expect(value).toBe(expectedValue)
-          }
+    "$it",
+    async ({
+      identity,
+      environmentFlags,
+      posthogFlags,
+      licenseFlags,
+      expected,
+      errorMessage,
+    }) => {
+      const env: Partial<typeof environment> = {
+        TENANT_FEATURE_FLAGS: environmentFlags,
+      }
+
+      if (posthogFlags) {
+        mockPosthogFlags(posthogFlags)
+        env.POSTHOG_TOKEN = "test"
+        env.POSTHOG_API_HOST = "https://us.i.posthog.com"
+        env.POSTHOG_PERSONAL_TOKEN = "test"
+      }
+
+      const ctx = { user: { license: { features: licenseFlags || [] } } }
+
+      await withEnv(env, async () => {
+        // We need to pass in node-fetch here otherwise nock won't get used
+        // because posthog-node uses axios under the hood.
+        init({
+          fetch: (url, opts) => {
+            return nodeFetch(url, opts)
+          },
         })
-      )
-  )
 
-  interface FailedTestCase {
-    tenant: string
-    flags: string
-    expected: string | RegExp
-  }
+        const fullIdentity: IdentityContext = {
+          _id: "us_1234",
+          tenantId: "default",
+          type: IdentityType.USER,
+          email: "test@example.com",
+          firstName: "Test",
+          lastName: "User",
+          ...identity,
+        }
 
-  it.each<FailedTestCase>([
-    {
-      tenant: "tenant1",
-      flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO",
-      expected: "Feature: FOO is not an allowed option",
-    },
-  ])(
-    "should fail with message \"$expected\" for $tenant with string '$flags'",
-    async ({ tenant, flags, expected }) => {
-      context.doInTenant(tenant, () =>
-        withFlags(flags, async () => {
-          await expect(fetch()).rejects.toThrow(expected)
+        await context.doInIdentityContext(fullIdentity, async () => {
+          if (errorMessage) {
+            await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow(
+              errorMessage
+            )
+          } else if (expected) {
+            const values = await flags.fetch(ctx as UserCtx)
+            expect(values).toMatchObject(expected)
+
+            for (const [key, expectedValue] of Object.entries(expected)) {
+              const value = await flags.get(
+                key as keyof typeof schema,
+                ctx as UserCtx
+              )
+              expect(value).toBe(expectedValue)
+            }
+          } else {
+            throw new Error("No expected value")
+          }
         })
-      )
+
+        shutdown()
+      })
     }
   )
+
+  it("should not error if PostHog is down", async () => {
+    const identity: IdentityContext = {
+      _id: "us_1234",
+      tenantId: "default",
+      type: IdentityType.USER,
+      email: "test@example.com",
+      firstName: "Test",
+      lastName: "User",
+    }
+
+    nock("https://us.i.posthog.com")
+      .post("/decide/?v=3", body => {
+        return body.token === "test" && body.distinct_id === "us_1234"
+      })
+      .reply(503)
+      .persist()
+
+    await withEnv(
+      { POSTHOG_TOKEN: "test", POSTHOG_API_HOST: "https://us.i.posthog.com" },
+      async () => {
+        await context.doInIdentityContext(identity, async () => {
+          await flags.fetch()
+        })
+      }
+    )
+  })
 })

package/src/index.ts

@@ -27,7 +27,7 @@ export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export * as timers from "./timers"
-export { default as env } from "./environment"
+export { default as env, withEnv, setEnv } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"