@budibase/backend-core 2.32.11 → 2.32.13

package/src/features/index.ts

@@ -1,281 +1,2 @@
-import env from "../environment"
-import * as context from "../context"
-import { PostHog, PostHogOptions } from "posthog-node"
-import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
-import tracer from "dd-trace"
-import { Duration } from "../utils"
-
-let posthog: PostHog | undefined
-export function init(opts?: PostHogOptions) {
-  if (
-    env.POSTHOG_TOKEN &&
-    env.POSTHOG_API_HOST &&
-    !env.SELF_HOSTED &&
-    env.POSTHOG_FEATURE_FLAGS_ENABLED
-  ) {
-    console.log("initializing posthog client...")
-    posthog = new PostHog(env.POSTHOG_TOKEN, {
-      host: env.POSTHOG_API_HOST,
-      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
-      featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
-      ...opts,
-    })
-  } else {
-    console.log("posthog disabled")
-  }
-}
-
-export function shutdown() {
-  posthog?.shutdown()
-}
-
-export abstract class Flag<T> {
-  static boolean(defaultValue: boolean): Flag<boolean> {
-    return new BooleanFlag(defaultValue)
-  }
-
-  static string(defaultValue: string): Flag<string> {
-    return new StringFlag(defaultValue)
-  }
-
-  static number(defaultValue: number): Flag<number> {
-    return new NumberFlag(defaultValue)
-  }
-
-  protected constructor(public defaultValue: T) {}
-
-  abstract parse(value: any): T
-}
-
-type UnwrapFlag<F> = F extends Flag<infer U> ? U : never
-
-export type FlagValues<T> = {
-  [K in keyof T]: UnwrapFlag<T[K]>
-}
-
-type KeysOfType<T, U> = {
-  [K in keyof T]: T[K] extends Flag<U> ? K : never
-}[keyof T]
-
-class BooleanFlag extends Flag<boolean> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return ["true", "t", "1"].includes(value.toLowerCase())
-    }
-
-    if (typeof value === "boolean") {
-      return value
-    }
-
-    throw new Error(`could not parse value "${value}" as boolean`)
-  }
-}
-
-class StringFlag extends Flag<string> {
-  parse(value: any) {
-    if (typeof value === "string") {
-      return value
-    }
-    throw new Error(`could not parse value "${value}" as string`)
-  }
-}
-
-class NumberFlag extends Flag<number> {
-  parse(value: any) {
-    if (typeof value === "number") {
-      return value
-    }
-
-    if (typeof value === "string") {
-      const parsed = parseFloat(value)
-      if (!isNaN(parsed)) {
-        return parsed
-      }
-    }
-
-    throw new Error(`could not parse value "${value}" as number`)
-  }
-}
-
-export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
-  // This is used to safely cache flags sets in the current request context.
-  // Because multiple sets could theoretically exist, we don't want the cache of
-  // one to leak into another.
-  private readonly setId: string
-
-  constructor(private readonly flagSchema: T) {
-    this.setId = crypto.randomUUID()
-  }
-
-  defaults(): FlagValues<T> {
-    return Object.keys(this.flagSchema).reduce((acc, key) => {
-      const typedKey = key as keyof T
-      acc[typedKey] = this.flagSchema[key].defaultValue
-      return acc
-    }, {} as FlagValues<T>)
-  }
-
-  isFlagName(name: string | number | symbol): name is keyof T {
-    return this.flagSchema[name as keyof T] !== undefined
-  }
-
-  async get<K extends keyof T>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<FlagValues<T>[K]> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async isEnabled<K extends KeysOfType<T, boolean>>(
-    key: K,
-    ctx?: UserCtx
-  ): Promise<boolean> {
-    const flags = await this.fetch(ctx)
-    return flags[key]
-  }
-
-  async fetch(ctx?: UserCtx): Promise<FlagValues<T>> {
-    return await tracer.trace("features.fetch", async span => {
-      const cachedFlags = context.getFeatureFlags<FlagValues<T>>(this.setId)
-      if (cachedFlags) {
-        span?.addTags({ fromCache: true })
-        return cachedFlags
-      }
-
-      const tags: Record<string, any> = {}
-      const flagValues = this.defaults()
-      const currentTenantId = context.getTenantId()
-      const specificallySetFalse = new Set<string>()
-
-      const split = (env.TENANT_FEATURE_FLAGS || "")
-        .split(",")
-        .map(x => x.split(":"))
-      for (const [tenantId, ...features] of split) {
-        if (!tenantId || (tenantId !== "*" && tenantId !== currentTenantId)) {
-          continue
-        }
-
-        tags[`readFromEnvironmentVars`] = true
-
-        for (let feature of features) {
-          let value = true
-          if (feature.startsWith("!")) {
-            feature = feature.slice(1)
-            value = false
-            specificallySetFalse.add(feature)
-          }
-
-          // ignore unknown flags
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (typeof flagValues[feature] !== "boolean") {
-            throw new Error(`Feature: ${feature} is not a boolean`)
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature as keyof FlagValues] = value
-          tags[`flags.${feature}.source`] = "environment"
-        }
-      }
-
-      const license = ctx?.user?.license
-      if (license) {
-        tags[`readFromLicense`] = true
-
-        for (const feature of license.features) {
-          if (!this.isFlagName(feature)) {
-            continue
-          }
-
-          if (
-            flagValues[feature] === true ||
-            specificallySetFalse.has(feature)
-          ) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          // @ts-expect-error - TS does not like you writing into a generic type,
-          // but we know that it's okay in this case because it's just an object.
-          flagValues[feature] = true
-          tags[`flags.${feature}.source`] = "license"
-        }
-      }
-
-      const identity = context.getIdentity()
-      tags[`identity.type`] = identity?.type
-      tags[`identity.tenantId`] = identity?.tenantId
-      tags[`identity._id`] = identity?._id
-
-      if (posthog && identity?.type === IdentityType.USER) {
-        tags[`readFromPostHog`] = true
-
-        const personProperties: Record<string, string> = {}
-        if (identity.tenantId) {
-          personProperties.tenantId = identity.tenantId
-        }
-
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(
-          identity._id,
-          {
-            personProperties,
-          }
-        )
-
-        for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
-          if (!this.isFlagName(name)) {
-            // We don't want an unexpected PostHog flag to break the app, so we
-            // just log it and continue.
-            console.warn(`Unexpected posthog flag "${name}": ${value}`)
-            continue
-          }
-
-          if (flagValues[name] === true || specificallySetFalse.has(name)) {
-            // If the flag is already set to through environment variables, we
-            // don't want to override it back to false here.
-            continue
-          }
-
-          const payload = posthogFlags.featureFlagPayloads?.[name]
-          const flag = this.flagSchema[name]
-          try {
-            // @ts-expect-error - TS does not like you writing into a generic
-            // type, but we know that it's okay in this case because it's just
-            // an object.
-            flagValues[name] = flag.parse(payload || value)
-            tags[`flags.${name}.source`] = "posthog"
-          } catch (err) {
-            // We don't want an invalid PostHog flag to break the app, so we just
-            // log it and continue.
-            console.warn(`Error parsing posthog flag "${name}": ${value}`, err)
-          }
-        }
-      }
-
-      context.setFeatureFlags(this.setId, flagValues)
-      for (const [key, value] of Object.entries(flagValues)) {
-        tags[`flags.${key}.value`] = value
-      }
-      span?.addTags(tags)
-
-      return flagValues
-    })
-  }
-}
-
-// This is the primary source of truth for feature flags. If you want to add a
-// new flag, add it here and use the `fetch` and `get` functions to access it.
-// All of the machinery in this file is to make sure that flags have their
-// default values set correctly and their types flow through the system.
-export const flags = new FlagSet({
-  DEFAULT_VALUES: Flag.boolean(env.isDev()),
-  AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-})
+export * from "./features"
+export * as testutils from "./tests/utils"

package/src/features/tests/utils.ts

@@ -0,0 +1,64 @@
+import { FeatureFlags, parseEnvFlags } from ".."
+import { setEnv } from "../../environment"
+
+function getCurrentFlags(): Record<string, Record<string, boolean>> {
+  const result: Record<string, Record<string, boolean>> = {}
+  for (const { tenantId, key, value } of parseEnvFlags(
+    process.env.TENANT_FEATURE_FLAGS || ""
+  )) {
+    const tenantFlags = result[tenantId] || {}
+    // Don't allow overwriting specifically false flags, to match the beheaviour
+    // of FlagSet.
+    if (tenantFlags[key] === false) {
+      continue
+    }
+    tenantFlags[key] = value
+    result[tenantId] = tenantFlags
+  }
+  return result
+}
+
+function buildFlagString(
+  flags: Record<string, Record<string, boolean>>
+): string {
+  const parts: string[] = []
+  for (const [tenantId, tenantFlags] of Object.entries(flags)) {
+    for (const [key, value] of Object.entries(tenantFlags)) {
+      if (value === false) {
+        parts.push(`${tenantId}:!${key}`)
+      } else {
+        parts.push(`${tenantId}:${key}`)
+      }
+    }
+  }
+  return parts.join(",")
+}
+
+export function setFeatureFlags(
+  tenantId: string,
+  flags: Partial<FeatureFlags>
+): () => void {
+  const current = getCurrentFlags()
+  for (const [key, value] of Object.entries(flags)) {
+    const tenantFlags = current[tenantId] || {}
+    tenantFlags[key] = value
+    current[tenantId] = tenantFlags
+  }
+  const flagString = buildFlagString(current)
+  return setEnv({ TENANT_FEATURE_FLAGS: flagString })
+}
+
+export function withFeatureFlags<T>(
+  tenantId: string,
+  flags: Partial<FeatureFlags>,
+  f: () => T
+) {
+  const cleanup = setFeatureFlags(tenantId, flags)
+  const result = f()
+  if (result instanceof Promise) {
+    return result.finally(cleanup)
+  } else {
+    cleanup()
+    return result
+  }
+}

package/src/middleware/passport/sso/sso.ts

@@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
 import { authError } from "../utils"
 import * as users from "../../../users"
 import * as context from "../../../context"
-import fetch from "node-fetch"
 import {
   SaveSSOUserFunction,
   SSOAuthDetails,
@@ -97,28 +96,13 @@ export async function authenticate(
   return done(null, ssoUser)
 }
 
-async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
-  const pictureUrl = details.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        return pictureUrl
-      }
-    }
-  }
-}
-
 /**
  * @returns a user that has been sync'd with third party information
  */
 async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
   let firstName
   let lastName
-  let pictureUrl
   let oauth2
-  let thirdPartyProfile
 
   if (details.profile) {
     const profile = details.profile
@@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
         lastName = name.familyName
       }
     }
-
-    pictureUrl = await getProfilePictureUrl(user, details)
-
-    thirdPartyProfile = {
-      ...profile._json,
-    }
   }
 
   // oauth tokens for future use
@@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
     providerType: details.providerType,
     firstName,
     lastName,
-    thirdPartyProfile,
-    pictureUrl,
     oauth2,
   }
 }

package/src/sql/sql.ts

@@ -139,29 +139,61 @@ class InternalBuilder {
     return this.table.schema[column]
   }
 
-  // Takes a string like foo and returns a quoted string like [foo] for SQL Server
-  // and "foo" for Postgres.
-  private quote(str: string): string {
+  private quoteChars(): [string, string] {
     switch (this.client) {
-      case SqlClient.SQL_LITE:
       case SqlClient.ORACLE:
       case SqlClient.POSTGRES:
-        return `"${str}"`
+        return ['"', '"']
       case SqlClient.MS_SQL:
-        return `[${str}]`
+        return ["[", "]"]
       case SqlClient.MARIADB:
       case SqlClient.MY_SQL:
-        return `\`${str}\``
+      case SqlClient.SQL_LITE:
+        return ["`", "`"]
+    }
+  }
+
+  // Takes a string like foo and returns a quoted string like [foo] for SQL Server
+  // and "foo" for Postgres.
+  private quote(str: string): string {
+    const [start, end] = this.quoteChars()
+    return `${start}${str}${end}`
+  }
+
+  private isQuoted(key: string): boolean {
+    const [start, end] = this.quoteChars()
+    return key.startsWith(start) && key.endsWith(end)
+  }
+
+  // Takes a string like a.b.c or an array like ["a", "b", "c"] and returns a
+  // quoted identifier like [a].[b].[c] for SQL Server and `a`.`b`.`c` for
+  // MySQL.
+  private quotedIdentifier(key: string | string[]): string {
+    if (!Array.isArray(key)) {
+      key = this.splitIdentifier(key)
     }
+    return key.map(part => this.quote(part)).join(".")
   }
 
-  // Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c]
-  // for SQL Server and `a`.`b`.`c` for MySQL.
-  private quotedIdentifier(key: string): string {
-    return key
-      .split(".")
-      .map(part => this.quote(part))
-      .join(".")
+  // Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"]
+  private splitIdentifier(key: string): string[] {
+    const [start, end] = this.quoteChars()
+    if (this.isQuoted(key)) {
+      return key.slice(1, -1).split(`${end}.${start}`)
+    }
+    return key.split(".")
+  }
+
+  private qualifyIdentifier(key: string): string {
+    const tableName = this.getTableName()
+    const parts = this.splitIdentifier(key)
+    if (parts[0] !== tableName) {
+      parts.unshift(tableName)
+    }
+    if (this.isQuoted(key)) {
+      return this.quotedIdentifier(parts)
+    }
+    return parts.join(".")
   }
 
   private isFullSelectStatementRequired(): boolean {
@@ -231,8 +263,13 @@ class InternalBuilder {
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
   // so when we use them we need to wrap them in to_char(). This function
   // converts a field name to the appropriate identifier.
-  private convertClobs(field: string): string {
-    const parts = field.split(".")
+  private convertClobs(field: string, opts?: { forSelect?: boolean }): string {
+    if (this.client !== SqlClient.ORACLE) {
+      throw new Error(
+        "you've called convertClobs on a DB that's not Oracle, this is a mistake"
+      )
+    }
+    const parts = this.splitIdentifier(field)
     const col = parts.pop()!
     const schema = this.table.schema[col]
     let identifier = this.quotedIdentifier(field)
@@ -244,7 +281,11 @@ class InternalBuilder {
       schema.type === FieldType.OPTIONS ||
       schema.type === FieldType.BARCODEQR
     ) {
-      identifier = `to_char(${identifier})`
+      if (opts?.forSelect) {
+        identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}`
+      } else {
+        identifier = `to_char(${identifier})`
+      }
     }
     return identifier
   }
@@ -859,28 +900,58 @@ class InternalBuilder {
     const fields = this.query.resource?.fields || []
     const tableName = this.getTableName()
     if (fields.length > 0) {
-      query = query.groupBy(fields.map(field => `${tableName}.${field}`))
-      query = query.select(fields.map(field => `${tableName}.${field}`))
+      const qualifiedFields = fields.map(field => this.qualifyIdentifier(field))
+      if (this.client === SqlClient.ORACLE) {
+        const groupByFields = qualifiedFields.map(field =>
+          this.convertClobs(field)
+        )
+        const selectFields = qualifiedFields.map(field =>
+          this.convertClobs(field, { forSelect: true })
+        )
+        query = query
+          .groupByRaw(groupByFields.join(", "))
+          .select(this.knex.raw(selectFields.join(", ")))
+      } else {
+        query = query.groupBy(qualifiedFields).select(qualifiedFields)
+      }
     }
     for (const aggregation of aggregations) {
       const op = aggregation.calculationType
-      const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
-      switch (op) {
-        case CalculationType.COUNT:
-          query = query.count(field)
-          break
-        case CalculationType.SUM:
-          query = query.sum(field)
-          break
-        case CalculationType.AVG:
-          query = query.avg(field)
-          break
-        case CalculationType.MIN:
-          query = query.min(field)
-          break
-        case CalculationType.MAX:
-          query = query.max(field)
-          break
+      if (op === CalculationType.COUNT) {
+        if ("distinct" in aggregation && aggregation.distinct) {
+          if (this.client === SqlClient.ORACLE) {
+            const field = this.convertClobs(`${tableName}.${aggregation.field}`)
+            query = query.select(
+              this.knex.raw(
+                `COUNT(DISTINCT ${field}) as ${this.quotedIdentifier(
+                  aggregation.name
+                )}`
+              )
+            )
+          } else {
+            query = query.countDistinct(
+              `${tableName}.${aggregation.field} as ${aggregation.name}`
+            )
+          }
+        } else {
+          query = query.count(`* as ${aggregation.name}`)
+        }
+      } else {
+        const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
+        switch (op) {
+          case CalculationType.SUM:
+            query = query.sum(field)
+            break
+          case CalculationType.AVG:
+            query = query.avg(field)
+            break
+          case CalculationType.MIN:
+            query = query.min(field)
+            break
+          case CalculationType.MAX:
+            query = query.max(field)
+            break
+        }
       }
     }
     return query

package/src/users/users.ts

@@ -24,6 +24,7 @@ import * as context from "../context"
 import { getGlobalDB } from "../context"
 import { isCreator } from "./utils"
 import { UserDB } from "./db"
+import { dataFilters } from "@budibase/shared-core"
 
 type GetOpts = { cleanup?: boolean }
 
@@ -262,10 +263,17 @@ export async function paginatedUsers({
     userList = await bulkGetGlobalUsersById(query?.oneOf?._id, {
       cleanup: true,
     })
+  } else if (query) {
+    // TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile
+    const response = await db.allDocs<User>(
+      getGlobalUserParams(null, { ...opts, limit: undefined })
+    )
+    userList = response.rows.map(row => row.doc!)
+    userList = dataFilters.search(userList, { query, limit: opts.limit }).rows
   } else {
     // no search, query allDocs
-    const response = await db.allDocs(getGlobalUserParams(null, opts))
-    userList = response.rows.map((row: any) => row.doc)
+    const response = await db.allDocs<User>(getGlobalUserParams(null, opts))
+    userList = response.rows.map(row => row.doc!)
   }
   return pagination(userList, pageSize, {
     paginate: true,