@budibase/backend-core 3.2.5 → 3.2.7

package/src/cache/tests/user.spec.ts

@@ -1,145 +0,0 @@
-import { User } from "@budibase/types"
-import { generator, structures } from "../../../tests"
-import { DBTestConfiguration } from "../../../tests/extra"
-import { getUsers } from "../user"
-import { getGlobalDB } from "../../context"
-import _ from "lodash"
-
-import * as redis from "../../redis/init"
-import { UserDB } from "../../users"
-
-const config = new DBTestConfiguration()
-
-describe("user cache", () => {
-  describe("getUsers", () => {
-    const users: User[] = []
-    beforeAll(async () => {
-      const userCount = 10
-      const userIds = generator.arrayOf(() => generator.guid(), {
-        min: userCount,
-        max: userCount,
-      })
-
-      await config.doInTenant(async () => {
-        const db = getGlobalDB()
-        for (const userId of userIds) {
-          const user = structures.users.user({ _id: userId })
-          await db.put(user)
-          users.push(user)
-        }
-      })
-    })
-
-    beforeEach(async () => {
-      jest.clearAllMocks()
-
-      const redisClient = await redis.getUserClient()
-      await redisClient.clear()
-    })
-
-    it("when no user is in cache, all of them are retrieved from db", async () => {
-      const usersToRequest = _.sampleSize(users, 5)
-
-      const userIdsToRequest = usersToRequest.map(x => x._id!)
-
-      jest.spyOn(UserDB, "bulkGet")
-
-      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
-
-      expect(results.users).toHaveLength(5)
-      expect(results).toEqual({
-        users: usersToRequest.map(u => ({
-          ...u,
-          budibaseAccess: true,
-          _rev: expect.any(String),
-        })),
-      })
-
-      expect(UserDB.bulkGet).toHaveBeenCalledTimes(1)
-      expect(UserDB.bulkGet).toHaveBeenCalledWith(userIdsToRequest)
-    })
-
-    it("on a second all, all of them are retrieved from cache", async () => {
-      const usersToRequest = _.sampleSize(users, 5)
-
-      const userIdsToRequest = usersToRequest.map(x => x._id!)
-
-      jest.spyOn(UserDB, "bulkGet")
-
-      await config.doInTenant(() => getUsers(userIdsToRequest))
-      const resultsFromCache = await config.doInTenant(() =>
-        getUsers(userIdsToRequest)
-      )
-
-      expect(resultsFromCache.users).toHaveLength(5)
-      expect(resultsFromCache).toEqual({
-        users: expect.arrayContaining(
-          usersToRequest.map(u => ({
-            ...u,
-            budibaseAccess: true,
-            _rev: expect.any(String),
-          }))
-        ),
-      })
-
-      expect(UserDB.bulkGet).toHaveBeenCalledTimes(1)
-    })
-
-    it("when some users are cached, only the missing ones are retrieved from db", async () => {
-      const usersToRequest = _.sampleSize(users, 5)
-
-      const userIdsToRequest = usersToRequest.map(x => x._id!)
-
-      jest.spyOn(UserDB, "bulkGet")
-
-      await config.doInTenant(() =>
-        getUsers([userIdsToRequest[0], userIdsToRequest[3]])
-      )
-      ;(UserDB.bulkGet as jest.Mock).mockClear()
-
-      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
-
-      expect(results.users).toHaveLength(5)
-      expect(results).toEqual({
-        users: expect.arrayContaining(
-          usersToRequest.map(u => ({
-            ...u,
-            budibaseAccess: true,
-            _rev: expect.any(String),
-          }))
-        ),
-      })
-
-      expect(UserDB.bulkGet).toHaveBeenCalledTimes(1)
-      expect(UserDB.bulkGet).toHaveBeenCalledWith([
-        userIdsToRequest[1],
-        userIdsToRequest[2],
-        userIdsToRequest[4],
-      ])
-    })
-
-    it("requesting existing and unexisting ids will return found ones", async () => {
-      const usersToRequest = _.sampleSize(users, 3)
-      const missingIds = [generator.guid(), generator.guid()]
-
-      const userIdsToRequest = _.shuffle([
-        ...missingIds,
-        ...usersToRequest.map(x => x._id!),
-      ])
-
-      const results = await config.doInTenant(() => getUsers(userIdsToRequest))
-
-      expect(results.users).toHaveLength(3)
-      expect(results).toEqual({
-        users: expect.arrayContaining(
-          usersToRequest.map(u => ({
-            ...u,
-            budibaseAccess: true,
-            _rev: expect.any(String),
-          }))
-        ),
-        notFoundIds: expect.arrayContaining(missingIds),
-      })
-    })
-  })
-})

package/src/cache/tests/writethrough.spec.ts

@@ -1,139 +0,0 @@
-import { DBTestConfiguration } from "../../../tests/extra"
-import { structures } from "../../../tests"
-import { Writethrough } from "../writethrough"
-import { getDB } from "../../db"
-import { Document } from "@budibase/types"
-import tk from "timekeeper"
-
-tk.freeze(Date.now())
-
-interface ValueDoc extends Document {
-  value: any
-}
-
-const DELAY = 5000
-
-describe("writethrough", () => {
-  const config = new DBTestConfiguration()
-
-  const db = getDB(structures.db.id())
-  const db2 = getDB(structures.db.id())
-
-  const writethrough = new Writethrough(db, DELAY)
-  const writethrough2 = new Writethrough(db2, DELAY)
-
-  const docId = structures.uuid()
-
-  beforeEach(() => {
-    jest.clearAllMocks()
-  })
-
-  describe("put", () => {
-    let current: any
-
-    it("should be able to store, will go to DB", async () => {
-      await config.doInTenant(async () => {
-        const response = await writethrough.put({
-          _id: docId,
-          value: 1,
-        })
-        const output = await db.get<any>(response.id)
-        current = output
-        expect(output.value).toBe(1)
-      })
-    })
-
-    it("second put shouldn't update DB", async () => {
-      await config.doInTenant(async () => {
-        const response = await writethrough.put({ ...current, value: 2 })
-        const output = await db.get<any>(response.id)
-        expect(current._rev).toBe(output._rev)
-        expect(output.value).toBe(1)
-      })
-    })
-
-    it("should put it again after delay period", async () => {
-      await config.doInTenant(async () => {
-        tk.freeze(Date.now() + DELAY + 1)
-        const response = await writethrough.put({ ...current, value: 3 })
-        const output = await db.get<any>(response.id)
-        expect(response.rev).not.toBe(current._rev)
-        expect(output.value).toBe(3)
-
-        current = output
-      })
-    })
-
-    it("should handle parallel DB updates ignoring conflicts", async () => {
-      await config.doInTenant(async () => {
-        tk.freeze(Date.now() + DELAY + 1)
-        const responses = await Promise.all([
-          writethrough.put({ ...current, value: 4 }),
-          writethrough.put({ ...current, value: 4 }),
-          writethrough.put({ ...current, value: 4 }),
-        ])
-
-        // with a lock, this will work
-        const newRev = responses.map(x => x.rev).find(x => x !== current._rev)
-        expect(newRev).toBeDefined()
-        expect(responses.map(x => x.rev)).toEqual(
-          expect.arrayContaining([current._rev, current._rev, newRev])
-        )
-
-        const output = await db.get<any>(current._id)
-        expect(output.value).toBe(4)
-        expect(output._rev).toBe(newRev)
-
-        current = output
-      })
-    })
-
-    it("should handle updates with documents falling behind", async () => {
-      await config.doInTenant(async () => {
-        tk.freeze(Date.now() + DELAY + 1)
-
-        const id = structures.uuid()
-        await writethrough.put({ _id: id, value: 1 })
-        const doc = await writethrough.get(id)
-
-        // Updating document
-        tk.freeze(Date.now() + DELAY + 1)
-        await writethrough.put({ ...doc, value: 2 })
-
-        // Update with the old rev value
-        tk.freeze(Date.now() + DELAY + 1)
-        const res = await writethrough.put({
-          ...doc,
-          value: 3,
-        })
-        expect(res.ok).toBe(true)
-
-        const output = await db.get<any>(id)
-        expect(output.value).toBe(3)
-        expect(output._rev).toBe(res.rev)
-      })
-    })
-  })
-
-  describe("get", () => {
-    it("should be able to retrieve", async () => {
-      await config.doInTenant(async () => {
-        const response = await writethrough.get<ValueDoc>(docId)
-        expect(response.value).toBe(4)
-      })
-    })
-  })
-
-  describe("same doc, different databases (tenancy)", () => {
-    it("should be able to two different databases", async () => {
-      await config.doInTenant(async () => {
-        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
-        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
-        expect(resp1.rev).toBeDefined()
-        expect(resp2.rev).toBeDefined()
-        expect((await db.get<any>("db1")).value).toBe("first")
-        expect((await db2.get<any>("db1")).value).toBe("second")
-      })
-    })
-  })
-})

package/src/cache/user.ts

@@ -1,154 +0,0 @@
-import * as redis from "../redis/init"
-import * as tenancy from "../tenancy"
-import * as context from "../context"
-import * as platform from "../platform"
-import env from "../environment"
-import * as accounts from "../accounts"
-import { UserDB } from "../users"
-import { sdk } from "@budibase/shared-core"
-import { User, UserMetadata } from "@budibase/types"
-
-const EXPIRY_SECONDS = 3600
-
-/**
- * The default populate user function
- */
-async function populateFromDB(userId: string, tenantId: string) {
-  const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get<UserMetadata>(userId)
-  user.budibaseAccess = true
-  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-    const account = await accounts.getAccount(user.email)
-    if (account) {
-      user.account = account
-      user.accountPortalAccess = true
-    }
-  }
-
-  return user
-}
-
-async function populateUsersFromDB(
-  userIds: string[]
-): Promise<{ users: User[]; notFoundIds?: string[] }> {
-  const getUsersResponse = await UserDB.bulkGet(userIds)
-
-  // Handle missed user ids
-  const notFoundIds = userIds.filter((uid, i) => !getUsersResponse[i])
-
-  const users = getUsersResponse.filter(x => x)
-
-  await Promise.all(
-    users.map(async (user: any) => {
-      user.budibaseAccess = true
-      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-        const account = await accounts.getAccount(user.email)
-        if (account) {
-          user.account = account
-          user.accountPortalAccess = true
-        }
-      }
-    })
-  )
-
-  if (notFoundIds.length) {
-    return { users, notFoundIds }
-  }
-  return { users }
-}
-
-/**
- * Get the requested user by id.
- * Use redis cache to first read the user.
- * If not present fallback to loading the user directly and re-caching.
- * @param userId the id of the user to get
- * @param tenantId the tenant of the user to get
- * @param email the email of the user to populate from account if needed
- * @param populateUser function to provide the user for re-caching. default to couch db
- * @returns
- */
-export async function getUser({
-  userId,
-  tenantId,
-  email,
-  populateUser,
-}: {
-  userId: string
-  email?: string
-  tenantId?: string
-  populateUser?: (
-    userId: string,
-    tenantId: string,
-    email?: string
-  ) => Promise<User>
-}) {
-  if (!populateUser) {
-    populateUser = populateFromDB
-  }
-  if (!tenantId) {
-    try {
-      tenantId = context.getTenantId()
-    } catch (err) {
-      tenantId = await platform.users.lookupTenantId(userId)
-    }
-  }
-  const client = await redis.getUserClient()
-  // try cache
-  let user: User = await client.get(userId)
-  if (!user) {
-    user = await populateUser(userId, tenantId, email)
-    await client.store(userId, user, EXPIRY_SECONDS)
-  }
-  if (user && !user.tenantId && tenantId) {
-    // make sure the tenant ID is always correct/set
-    user.tenantId = tenantId
-  }
-  // if has groups, could have builder permissions granted by a group
-  if (user.userGroups && !sdk.users.isGlobalBuilder(user)) {
-    await context.doInTenant(tenantId, async () => {
-      const appIds = await UserDB.getGroupBuilderAppIds(user)
-      if (appIds.length) {
-        const existing = user.builder?.apps || []
-        user.builder = {
-          apps: [...new Set(existing.concat(appIds))],
-        }
-      }
-    })
-  }
-  return user
-}
-
-/**
- * Get the requested users by id.
- * Use redis cache to first read the users.
- * If not present fallback to loading the users directly and re-caching.
- * @param userIds the ids of the user to get
- * @param tenantId the tenant of the users to get
- * @returns
- */
-export async function getUsers(
-  userIds: string[]
-): Promise<{ users: User[]; notFoundIds?: string[] }> {
-  const client = await redis.getUserClient()
-  // try cache
-  let usersFromCache = await client.bulkGet<User>(userIds)
-  const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
-  const users = Object.values(usersFromCache)
-  let notFoundIds
-
-  if (missingUsersFromCache.length) {
-    const usersFromDb = await populateUsersFromDB(missingUsersFromCache)
-
-    notFoundIds = usersFromDb.notFoundIds
-    for (const userToCache of usersFromDb.users) {
-      await client.store(userToCache._id!, userToCache, EXPIRY_SECONDS)
-    }
-    users.push(...usersFromDb.users)
-  }
-  return { users, notFoundIds: notFoundIds }
-}
-
-export async function invalidateUser(userId: string) {
-  const client = await redis.getUserClient()
-  await client.delete(userId)
-}

package/src/cache/writethrough.ts

@@ -1,133 +0,0 @@
-import BaseCache from "./base"
-import { getWritethroughClient } from "../redis/init"
-import { logWarn } from "../logging"
-import { Database, Document, LockName, LockType } from "@budibase/types"
-import * as locks from "../redis/redlockImpl"
-
-const DEFAULT_WRITE_RATE_MS = 10000
-let CACHE: BaseCache | null = null
-
-interface CacheItem<T extends Document> {
-  doc: T
-  lastWrite: number
-}
-
-async function getCache() {
-  if (!CACHE) {
-    const client = await getWritethroughClient()
-    CACHE = new BaseCache(client)
-  }
-  return CACHE
-}
-
-function makeCacheKey(db: Database, key: string) {
-  return db.name + key
-}
-
-function makeCacheItem<T extends Document>(
-  doc: T,
-  lastWrite: number | null = null
-): CacheItem<T> {
-  return { doc, lastWrite: lastWrite || Date.now() }
-}
-
-async function put(
-  db: Database,
-  doc: Document,
-  writeRateMs: number = DEFAULT_WRITE_RATE_MS
-) {
-  const cache = await getCache()
-  const key = doc._id
-  let cacheItem: CacheItem<any> | undefined
-  if (key) {
-    cacheItem = await cache.get(makeCacheKey(db, key))
-  }
-  const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs
-  let output = doc
-  if (updateDb) {
-    const lockResponse = await locks.doWithLock(
-      {
-        type: LockType.TRY_ONCE,
-        name: LockName.PERSIST_WRITETHROUGH,
-        resource: key,
-        ttl: 15000,
-      },
-      async () => {
-        const writeDb = async (toWrite: any) => {
-          // doc should contain the _id and _rev
-          const response = await db.put(toWrite, { force: true })
-          output._id = response.id
-          output._rev = response.rev
-        }
-        try {
-          await writeDb(doc)
-        } catch (err: any) {
-          if (err.status !== 409) {
-            throw err
-          } else {
-            // Swallow 409s but log them
-            logWarn(`Ignoring conflict in write-through cache`)
-          }
-        }
-      }
-    )
-
-    if (!lockResponse.executed) {
-      logWarn(`Ignoring redlock conflict in write-through cache`)
-    }
-  }
-  // if we are updating the DB then need to set the lastWrite to now
-  cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite)
-  if (output._id) {
-    await cache.store(makeCacheKey(db, output._id), cacheItem)
-  }
-  return { ok: true, id: output._id, rev: output._rev }
-}
-
-async function get<T extends Document>(db: Database, id: string): Promise<T> {
-  const cache = await getCache()
-  const cacheKey = makeCacheKey(db, id)
-  let cacheItem: CacheItem<T> = await cache.get(cacheKey)
-  if (!cacheItem) {
-    const doc = await db.get<T>(id)
-    cacheItem = makeCacheItem(doc)
-    await cache.store(cacheKey, cacheItem)
-  }
-  return cacheItem.doc
-}
-
-async function remove(db: Database, docOrId: any, rev?: any): Promise<void> {
-  const cache = await getCache()
-  if (!docOrId) {
-    throw new Error("No ID/Rev provided.")
-  }
-  const id = typeof docOrId === "string" ? docOrId : docOrId._id
-  rev = typeof docOrId === "string" ? rev : docOrId._rev
-  try {
-    await cache.delete(makeCacheKey(db, id))
-  } finally {
-    await db.remove(id, rev)
-  }
-}
-
-export class Writethrough {
-  db: Database
-  writeRateMs: number
-
-  constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
-    this.db = db
-    this.writeRateMs = writeRateMs
-  }
-
-  async put(doc: any, writeRateMs: number = this.writeRateMs) {
-    return put(this.db, doc, writeRateMs)
-  }
-
-  async get<T extends Document>(id: string) {
-    return get<T>(this.db, id)
-  }
-
-  async remove(docOrId: any, rev?: any) {
-    return remove(this.db, docOrId, rev)
-  }
-}