@budibase/backend-core 3.2.4 → 3.2.6

package/src/cache/tests/writethrough.spec.ts

@@ -1,139 +0,0 @@
-import { DBTestConfiguration } from "../../../tests/extra"
-import { structures } from "../../../tests"
-import { Writethrough } from "../writethrough"
-import { getDB } from "../../db"
-import { Document } from "@budibase/types"
-import tk from "timekeeper"
-
-tk.freeze(Date.now())
-
-interface ValueDoc extends Document {
-  value: any
-}
-
-const DELAY = 5000
-
-describe("writethrough", () => {
-  const config = new DBTestConfiguration()
-
-  const db = getDB(structures.db.id())
-  const db2 = getDB(structures.db.id())
-
-  const writethrough = new Writethrough(db, DELAY)
-  const writethrough2 = new Writethrough(db2, DELAY)
-
-  const docId = structures.uuid()
-
-  beforeEach(() => {
-    jest.clearAllMocks()
-  })
-
-  describe("put", () => {
-    let current: any
-
-    it("should be able to store, will go to DB", async () => {
-      await config.doInTenant(async () => {
-        const response = await writethrough.put({
-          _id: docId,
-          value: 1,
-        })
-        const output = await db.get<any>(response.id)
-        current = output
-        expect(output.value).toBe(1)
-      })
-    })
-
-    it("second put shouldn't update DB", async () => {
-      await config.doInTenant(async () => {
-        const response = await writethrough.put({ ...current, value: 2 })
-        const output = await db.get<any>(response.id)
-        expect(current._rev).toBe(output._rev)
-        expect(output.value).toBe(1)
-      })
-    })
-
-    it("should put it again after delay period", async () => {
-      await config.doInTenant(async () => {
-        tk.freeze(Date.now() + DELAY + 1)
-        const response = await writethrough.put({ ...current, value: 3 })
-        const output = await db.get<any>(response.id)
-        expect(response.rev).not.toBe(current._rev)
-        expect(output.value).toBe(3)
-
-        current = output
-      })
-    })
-
-    it("should handle parallel DB updates ignoring conflicts", async () => {
-      await config.doInTenant(async () => {
-        tk.freeze(Date.now() + DELAY + 1)
-        const responses = await Promise.all([
-          writethrough.put({ ...current, value: 4 }),
-          writethrough.put({ ...current, value: 4 }),
-          writethrough.put({ ...current, value: 4 }),
-        ])
-
-        // with a lock, this will work
-        const newRev = responses.map(x => x.rev).find(x => x !== current._rev)
-        expect(newRev).toBeDefined()
-        expect(responses.map(x => x.rev)).toEqual(
-          expect.arrayContaining([current._rev, current._rev, newRev])
-        )
-
-        const output = await db.get<any>(current._id)
-        expect(output.value).toBe(4)
-        expect(output._rev).toBe(newRev)
-
-        current = output
-      })
-    })
-
-    it("should handle updates with documents falling behind", async () => {
-      await config.doInTenant(async () => {
-        tk.freeze(Date.now() + DELAY + 1)
-
-        const id = structures.uuid()
-        await writethrough.put({ _id: id, value: 1 })
-        const doc = await writethrough.get(id)
-
-        // Updating document
-        tk.freeze(Date.now() + DELAY + 1)
-        await writethrough.put({ ...doc, value: 2 })
-
-        // Update with the old rev value
-        tk.freeze(Date.now() + DELAY + 1)
-        const res = await writethrough.put({
-          ...doc,
-          value: 3,
-        })
-        expect(res.ok).toBe(true)
-
-        const output = await db.get<any>(id)
-        expect(output.value).toBe(3)
-        expect(output._rev).toBe(res.rev)
-      })
-    })
-  })
-
-  describe("get", () => {
-    it("should be able to retrieve", async () => {
-      await config.doInTenant(async () => {
-        const response = await writethrough.get<ValueDoc>(docId)
-        expect(response.value).toBe(4)
-      })
-    })
-  })
-
-  describe("same doc, different databases (tenancy)", () => {
-    it("should be able to two different databases", async () => {
-      await config.doInTenant(async () => {
-        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
-        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
-        expect(resp1.rev).toBeDefined()
-        expect(resp2.rev).toBeDefined()
-        expect((await db.get<any>("db1")).value).toBe("first")
-        expect((await db2.get<any>("db1")).value).toBe("second")
-      })
-    })
-  })
-})

package/src/cache/user.ts

@@ -1,154 +0,0 @@
-import * as redis from "../redis/init"
-import * as tenancy from "../tenancy"
-import * as context from "../context"
-import * as platform from "../platform"
-import env from "../environment"
-import * as accounts from "../accounts"
-import { UserDB } from "../users"
-import { sdk } from "@budibase/shared-core"
-import { User, UserMetadata } from "@budibase/types"
-
-const EXPIRY_SECONDS = 3600
-
-/**
- * The default populate user function
- */
-async function populateFromDB(userId: string, tenantId: string) {
-  const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<UserMetadata>(userId)
-  user.budibaseAccess = true
-  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-    const account = await accounts.getAccount(user.email)
-    if (account) {
-      user.account = account
-      user.accountPortalAccess = true
-    }
-  }
-
-  return user
-}
-
-async function populateUsersFromDB(
-  userIds: string[]
-): Promise<{ users: User[]; notFoundIds?: string[] }> {
-  const getUsersResponse = await UserDB.bulkGet(userIds)
-
-  // Handle missed user ids
-  const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i])
-
-  const users = getUsersResponse.filter(x => x)
-
-  await Promise.all(
-    users.map(async (user: any) => {
-      user.budibaseAccess = true
-      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-        const account = await accounts.getAccount(user.email)
-        if (account) {
-          user.account = account
-          user.accountPortalAccess = true
-        }
-      }
-    })
-  )
-
-  if (notFoundIds.length) {
-    return { users, notFoundIds }
-  }
-  return { users }
-}
-
-/**
- * Get the requested user by id.
- * Use redis cache to first read the user.
- * If not present fallback to loading the user directly and re-caching.
- * @param userId the id of the user to get
- * @param tenantId the tenant of the user to get
- * @param email the email of the user to populate from account if needed
- * @param populateUser function to provide the user for re-caching. default to couch db
- * @returns
- */
-export async function getUser({
-  userId,
-  tenantId,
-  email,
-  populateUser,
-}: {
-  userId: string
-  email?: string
-  tenantId?: string
-  populateUser?: (
-    userId: string,
-    tenantId: string,
-    email?: string
-  ) => Promise<User>
-}) {
-  if (!populateUser) {
-    populateUser = populateFromDB
-  }
-  if (!tenantId) {
-    try {
-      tenantId = context.getTenantId()
-    } catch (err) {
-      tenantId = await platform.users.lookupTenantId(userId)
-    }
-  }
-  const client = await redis.getUserClient()
-  // try cache
-  let user: User = await client.get(userId)
-  if (!user) {
-    user = await populateUser(userId, tenantId, email)
-    await client.store(userId, user, EXPIRY_SECONDS)
-  }
-  if (user && !user.tenantId && tenantId) {
-    // make sure the tenant ID is always correct/set
-    user.tenantId = tenantId
-  }
-  // if has groups, could have builder permissions granted by a group
-  if (user.userGroups && !sdk.users.isGlobalBuilder(user)) {
-    await context.doInTenant(tenantId, async () => {
-      const appIds = await UserDB.getGroupBuilderAppIds(user)
-      if (appIds.length) {
-        const existing = user.builder?.apps || []
-        user.builder = {
-          apps: [...new Set(existing.concat(appIds))],
-        }
-      }
-    })
-  }
-  return user
-}
-
-/**
- * Get the requested users by id.
- * Use redis cache to first read the users.
- * If not present fallback to loading the users directly and re-caching.
- * @param userIds the ids of the user to get
- * @param tenantId the tenant of the users to get
- * @returns
- */
-export async function getUsers(
-  userIds: string[]
-): Promise<{ users: User[]; notFoundIds?: string[] }> {
-  const client = await redis.getUserClient()
-  // try cache
-  let usersFromCache = await client.bulkGet<User>(userIds)
-  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
-  const users = Object.values(usersFromCache)
-  let notFoundIds
-
-  if (missingUsersFromCache.length) {
-    const usersFromDb = await populateUsersFromDB(missingUsersFromCache)
-
-    notFoundIds = usersFromDb.notFoundIds
-    for (const userToCache of usersFromDb.users) {
-      await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS)
-    }
-    users.push(...usersFromDb.users)
-  }
-  return { users, notFoundIds: notFoundIds }
-}
-
-export async function invalidateUser(userId: string) {
-  const client = await redis.getUserClient()
-  await client.delete(userId)
-}

package/src/cache/writethrough.ts

@@ -1,133 +0,0 @@
-import BaseCache from "./base"
-import { getWritethroughClient } from "../redis/init"
-import { logWarn } from "../logging"
-import { Database, Document, LockName, LockType } from "@budibase/types"
-import * as locks from "../redis/redlockImpl"
-
-const DEFAULT_WRITE_RATE_MS = 10000
-let CACHE: BaseCache | null = null
-
-interface CacheItem<T extends Document> {
-  doc: T
-  lastWrite: number
-}
-
-async function getCache() {
-  if (!CACHE) {
-    const client = await getWritethroughClient()
-    CACHE = new BaseCache(client)
-  }
-  return CACHE
-}
-
-function makeCacheKey(db: Database, key: string) {
-  return db.name + key
-}
-
-function makeCacheItem<T extends Document>(
-  doc: T,
-  lastWrite: number | null = null
-): CacheItem<T> {
-  return { doc, lastWrite: lastWrite || Date.now() }
-}
-
-async function put(
-  db: Database,
-  doc: Document,
-  writeRateMs: number = DEFAULT_WRITE_RATE_MS
-) {
-  const cache = await getCache()
-  const key = doc._id
-  let cacheItem: CacheItem<any> | undefined
-  if (key) {
-    cacheItem = await cache.get(makeCacheKey(db, key))
-  }
-  const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs
-  let output = doc
-  if (updateDb) {
-    const lockResponse = await locks.doWithLock(
-      {
-        type: LockType.TRY_ONCE,
-        name: LockName.PERSIST_WRITETHROUGH,
-        resource: key,
-        ttl: 15000,
-      },
-      async () => {
-        const writeDb = async (toWrite: any) => {
-          // doc should contain the _id and _rev
-          const response = await db.put(toWrite, { force: true })
-          output._id = response.id
-          output._rev = response.rev
-        }
-        try {
-          await writeDb(doc)
-        } catch (err: any) {
-          if (err.status !== 409) {
-            throw err
-          } else {
-            // Swallow 409s but log them
-            logWarn(`Ignoring conflict in write-through cache`)
-          }
-        }
-      }
-    )
-
-    if (!lockResponse.executed) {
-      logWarn(`Ignoring redlock conflict in write-through cache`)
-    }
-  }
-  // if we are updating the DB then need to set the lastWrite to now
-  cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite)
-  if (output._id) {
-    await cache.store(makeCacheKey(db, output._id), cacheItem)
-  }
-  return { ok: true, id: output._id, rev: output._rev }
-}
-
-async function get<T extends Document>(db: Database, id: string): Promise<T> {
-  const cache = await getCache()
-  const cacheKey = makeCacheKey(db, id)
-  let cacheItem: CacheItem<T> = await cache.get(cacheKey)
-  if (!cacheItem) {
-    const doc = await db.get<T>(id)
-    cacheItem = makeCacheItem(doc)
-    await cache.store(cacheKey, cacheItem)
-  }
-  return cacheItem.doc
-}
-
-async function remove(db: Database, docOrId: any, rev?: any): Promise<void> {
-  const cache = await getCache()
-  if (!docOrId) {
-    throw new Error("No ID/Rev provided.")
-  }
-  const id = typeof docOrId === "string" ? docOrId : docOrId._id
-  rev = typeof docOrId === "string" ? rev : docOrId._rev
-  try {
-    await cache.delete(makeCacheKey(db, id))
-  } finally {
-    await db.remove(id, rev)
-  }
-}
-
-export class Writethrough {
-  db: Database
-  writeRateMs: number
-
-  constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
-    this.db = db
-    this.writeRateMs = writeRateMs
-  }
-
-  async put(doc: any, writeRateMs: number = this.writeRateMs) {
-    return put(this.db, doc, writeRateMs)
-  }
-
-  async get<T extends Document>(id: string) {
-    return get<T>(this.db, id)
-  }
-
-  async remove(docOrId: any, rev?: any) {
-    return remove(this.db, docOrId, rev)
-  }
-}

package/src/configs/configs.ts

@@ -1,263 +0,0 @@
-import {
-  AIConfig,
-  Config,
-  ConfigType,
-  GoogleConfig,
-  GoogleInnerConfig,
-  OIDCConfig,
-  OIDCInnerConfig,
-  OIDCLogosConfig,
-  SCIMConfig,
-  SCIMInnerConfig,
-  SettingsConfig,
-  SettingsInnerConfig,
-  SMTPConfig,
-  SMTPInnerConfig,
-} from "@budibase/types"
-import { DocumentType, SEPARATOR } from "../constants"
-import { CacheKey, TTL, withCache } from "../cache"
-import * as context from "../context"
-import env from "../environment"
-
-// UTILS
-
-/**
- * Generates a new configuration ID.
- * @returns The new configuration ID which the config doc can be stored under.
- */
-export function generateConfigID(type: ConfigType) {
-  return `${DocumentType.CONFIG}${SEPARATOR}${type}`
-}
-
-export async function getConfig<T extends Config>(
-  type: ConfigType
-): Promise<T | undefined> {
-  const db = context.getGlobalDB()
-  try {
-    // await to catch error
-    return (await db.get(generateConfigID(type))) as T
-  } catch (e: any) {
-    if (e.status === 404) {
-      return
-    }
-    throw e
-  }
-}
-
-export async function save(
-  config: Config
-): Promise<{ id: string; rev: string }> {
-  const db = context.getGlobalDB()
-  return db.put(config)
-}
-
-// SETTINGS
-
-export async function getSettingsConfigDoc(): Promise<SettingsConfig> {
-  let config = await getConfig<SettingsConfig>(ConfigType.SETTINGS)
-
-  if (!config) {
-    config = {
-      _id: generateConfigID(ConfigType.SETTINGS),
-      type: ConfigType.SETTINGS,
-      config: {},
-    }
-  }
-
-  // overridden fields
-  config.config.platformUrl = await getPlatformUrl({
-    tenantAware: true,
-    config: config.config,
-  })
-  config.config.analyticsEnabled = await analyticsEnabled({
-    config: config.config,
-  })
-
-  return config
-}
-
-export async function getSettingsConfig(): Promise<SettingsInnerConfig> {
-  return (await getSettingsConfigDoc()).config
-}
-
-export async function getPlatformUrl(
-  opts: { tenantAware: boolean; config?: SettingsInnerConfig } = {
-    tenantAware: true,
-  }
-) {
-  let platformUrl = env.PLATFORM_URL || "http://localhost:10000"
-
-  if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) {
-    // cloud and multi tenant - add the tenant to the default platform url
-    const tenantId = context.getTenantId()
-    if (!platformUrl.includes("localhost:")) {
-      platformUrl = platformUrl.replace("://", `://${tenantId}.`)
-    }
-  } else if (env.SELF_HOSTED) {
-    const config = opts?.config
-      ? opts.config
-      : // direct to db to prevent infinite loop
-        (await getConfig<SettingsConfig>(ConfigType.SETTINGS))?.config
-    if (config?.platformUrl) {
-      platformUrl = config.platformUrl
-    }
-  }
-
-  return platformUrl
-}
-
-export const analyticsEnabled = async (opts?: {
-  config?: SettingsInnerConfig
-}) => {
-  // cloud - always use the environment variable
-  if (!env.SELF_HOSTED) {
-    return !!env.ENABLE_ANALYTICS
-  }
-
-  // self host - prefer the settings doc
-  // use cache as events have high throughput
-  const enabledInDB = await withCache(
-    CacheKey.ANALYTICS_ENABLED,
-    TTL.ONE_DAY,
-    async () => {
-      const config = opts?.config
-        ? opts.config
-        : // direct to db to prevent infinite loop
-          (await getConfig<SettingsConfig>(ConfigType.SETTINGS))?.config
-
-      // need to do explicit checks in case the field is not set
-      if (config?.analyticsEnabled === false) {
-        return false
-      } else if (config?.analyticsEnabled === true) {
-        return true
-      }
-    }
-  )
-
-  if (enabledInDB !== undefined) {
-    return enabledInDB
-  }
-
-  // fallback to the environment variable
-  // explicitly check for 0 or false here, undefined or otherwise is treated as true
-  const envEnabled: any = env.ENABLE_ANALYTICS
-  if (envEnabled === 0 || envEnabled === false) {
-    return false
-  } else {
-    return true
-  }
-}
-
-// GOOGLE
-
-async function getGoogleConfigDoc(): Promise<GoogleConfig | undefined> {
-  return await getConfig<GoogleConfig>(ConfigType.GOOGLE)
-}
-
-export async function getGoogleConfig(): Promise<
-  GoogleInnerConfig | undefined
-> {
-  const config = await getGoogleConfigDoc()
-  return config?.config
-}
-
-export async function getGoogleDatasourceConfig(): Promise<
-  GoogleInnerConfig | undefined
-> {
-  if (!env.SELF_HOSTED) {
-    // always use the env vars in cloud
-    return getDefaultGoogleConfig()
-  }
-
-  // prefer the config in self-host
-  let config = await getGoogleConfig()
-
-  // fallback to env vars
-  if (!config || !config.activated) {
-    config = getDefaultGoogleConfig()
-  }
-
-  return config
-}
-
-export function getDefaultGoogleConfig(): GoogleInnerConfig | undefined {
-  if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) {
-    return {
-      clientID: env.GOOGLE_CLIENT_ID!,
-      clientSecret: env.GOOGLE_CLIENT_SECRET!,
-      activated: true,
-    }
-  }
-}
-
-// OIDC
-
-export async function getOIDCLogosDoc(): Promise<OIDCLogosConfig | undefined> {
-  return getConfig<OIDCLogosConfig>(ConfigType.OIDC_LOGOS)
-}
-
-async function getOIDCConfigDoc(): Promise<OIDCConfig | undefined> {
-  return getConfig<OIDCConfig>(ConfigType.OIDC)
-}
-
-export async function getOIDCConfig(): Promise<OIDCInnerConfig | undefined> {
-  const config = (await getOIDCConfigDoc())?.config
-  // default to the 0th config
-  return config?.configs && config.configs[0]
-}
-
-/**
- * @param configId The config id of the inner config to retrieve
- */
-export async function getOIDCConfigById(
-  configId: string
-): Promise<OIDCInnerConfig | undefined> {
-  const config = (await getConfig<OIDCConfig>(ConfigType.OIDC))?.config
-  return config && config.configs.filter((c: any) => c.uuid === configId)[0]
-}
-
-// SMTP
-
-export async function getSMTPConfigDoc(): Promise<SMTPConfig | undefined> {
-  return getConfig<SMTPConfig>(ConfigType.SMTP)
-}
-
-export async function getSMTPConfig(
-  isAutomation?: boolean
-): Promise<SMTPInnerConfig | undefined> {
-  const config = await getSMTPConfigDoc()
-  if (config) {
-    return config.config
-  }
-
-  // always allow fallback in self host
-  // in cloud don't allow for automations
-  const allowFallback = env.SELF_HOSTED || !isAutomation
-
-  // Use an SMTP fallback configuration from env variables
-  if (env.SMTP_FALLBACK_ENABLED && allowFallback) {
-    return {
-      port: env.SMTP_PORT,
-      host: env.SMTP_HOST!,
-      secure: false,
-      from: env.SMTP_FROM_ADDRESS!,
-      auth: {
-        user: env.SMTP_USER!,
-        pass: env.SMTP_PASSWORD!,
-      },
-    }
-  }
-}
-
-// SCIM
-
-export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
-  const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
-  return config?.config
-}
-
-// AI
-
-export async function getAIConfig(): Promise<AIConfig | undefined> {
-  return getConfig<AIConfig>(ConfigType.AI)
-}

package/src/configs/index.ts

@@ -1 +0,0 @@
-export * from "./configs"