@budibase/backend-core 2.9.40-alpha.6 → 2.10.1

package/src/cache/appMetadata.ts

@@ -0,0 +1,88 @@
+import { getAppClient } from "../redis/init"
+import { doWithDB, DocumentType } from "../db"
+import { Database, App } from "@budibase/types"
+
+export enum AppState {
+  INVALID = "invalid",
+}
+
+export interface DeletedApp {
+  state: AppState
+}
+
+const EXPIRY_SECONDS = 3600
+
+/**
+ * The default populate app metadata function
+ */
+async function populateFromDB(appId: string) {
+  return doWithDB(
+    appId,
+    (db: Database) => {
+      return db.get(DocumentType.APP_METADATA)
+    },
+    { skip_setup: true }
+  )
+}
+
+function isInvalid(metadata?: { state: string }) {
+  return !metadata || metadata.state === AppState.INVALID
+}
+
+/**
+ * Get the requested app metadata by id.
+ * Use redis cache to first read the app metadata.
+ * If not present fallback to loading the app metadata directly and re-caching.
+ * @param {string} appId the id of the app to get metadata from.
+ * @returns {object} the app metadata.
+ */
+export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
+  const client = await getAppClient()
+  // try cache
+  let metadata = await client.get(appId)
+  if (!metadata) {
+    let expiry: number | undefined = EXPIRY_SECONDS
+    try {
+      metadata = await populateFromDB(appId)
+    } catch (err: any) {
+      // app DB left around, but no metadata, it is invalid
+      if (err && err.status === 404) {
+        metadata = { state: AppState.INVALID }
+        // don't expire the reference to an invalid app, it'll only be
+        // updated if a metadata doc actually gets stored (app is remade/reverted)
+        expiry = undefined
+      } else {
+        throw err
+      }
+    }
+    // needed for cypress/some scenarios where the caching happens
+    // so quickly the requests can get slightly out of sync
+    // might store its invalid just before it stores its valid
+    if (isInvalid(metadata)) {
+      const temp = await client.get(appId)
+      if (temp) {
+        metadata = temp
+      }
+    }
+    await client.store(appId, metadata, expiry)
+  }
+
+  return metadata
+}
+
+/**
+ * Invalidate/reset the cached metadata when a change occurs in the db.
+ * @param appId {string} the cache key to bust/update.
+ * @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with.
+ * @return {Promise<void>} will respond with success when cache is updated.
+ */
+export async function invalidateAppMetadata(appId: string, newMetadata?: any) {
+  if (!appId) {
+    throw "Cannot invalidate if no app ID provided."
+  }
+  const client = await getAppClient()
+  await client.delete(appId)
+  if (newMetadata) {
+    await client.store(appId, newMetadata, EXPIRY_SECONDS)
+  }
+}

package/src/cache/base/index.ts

@@ -0,0 +1,92 @@
+import { getTenantId } from "../../context"
+import * as redis from "../../redis/init"
+import { Client } from "../../redis"
+
+function generateTenantKey(key: string) {
+  const tenantId = getTenantId()
+  return `${key}:${tenantId}`
+}
+
+export default class BaseCache {
+  client: Client | undefined
+
+  constructor(client: Client | undefined = undefined) {
+    this.client = client
+  }
+
+  async getClient() {
+    return !this.client ? await redis.getCacheClient() : this.client
+  }
+
+  async keys(pattern: string) {
+    const client = await this.getClient()
+    return client.keys(pattern)
+  }
+
+  /**
+   * Read only from the cache.
+   */
+  async get(key: string, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    return client.get(key)
+  }
+
+  /**
+   * Write to the cache.
+   */
+  async store(
+    key: string,
+    value: any,
+    ttl: number | null = null,
+    opts = { useTenancy: true }
+  ) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    await client.store(key, value, ttl)
+  }
+
+  /**
+   * Remove from cache.
+   */
+  async delete(key: string, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key
+    const client = await this.getClient()
+    return client.delete(key)
+  }
+
+  /**
+   * Read from the cache. Write to the cache if not exists.
+   */
+  async withCache(
+    key: string,
+    ttl: number,
+    fetchFn: any,
+    opts = { useTenancy: true }
+  ) {
+    const cachedValue = await this.get(key, opts)
+    if (cachedValue) {
+      return cachedValue
+    }
+
+    try {
+      const fetchedValue = await fetchFn()
+
+      await this.store(key, fetchedValue, ttl, opts)
+      return fetchedValue
+    } catch (err) {
+      console.error("Error fetching before cache - ", err)
+      throw err
+    }
+  }
+
+  async bustCache(key: string, opts = { client: null }) {
+    const client = await this.getClient()
+    try {
+      await client.delete(generateTenantKey(key))
+    } catch (err) {
+      console.error("Error busting cache - ", err)
+      throw err
+    }
+  }
+}

package/src/cache/generic.ts

@@ -0,0 +1,30 @@
+const BaseCache = require("./base")
+
+const GENERIC = new BaseCache.default()
+
+export enum CacheKey {
+  CHECKLIST = "checklist",
+  INSTALLATION = "installation",
+  ANALYTICS_ENABLED = "analyticsEnabled",
+  UNIQUE_TENANT_ID = "uniqueTenantId",
+  EVENTS = "events",
+  BACKFILL_METADATA = "backfillMetadata",
+  EVENTS_RATE_LIMIT = "eventsRateLimit",
+}
+
+export enum TTL {
+  ONE_MINUTE = 600,
+  ONE_HOUR = 3600,
+  ONE_DAY = 86400,
+}
+
+function performExport(funcName: string) {
+  return (...args: any) => GENERIC[funcName](...args)
+}
+
+export const keys = performExport("keys")
+export const get = performExport("get")
+export const store = performExport("store")
+export const destroy = performExport("delete")
+export const withCache = performExport("withCache")
+export const bustCache = performExport("bustCache")

package/src/cache/index.ts

@@ -0,0 +1,5 @@
+export * as generic from "./generic"
+export * as user from "./user"
+export * as app from "./appMetadata"
+export * as writethrough from "./writethrough"
+export * from "./generic"

package/src/cache/tests/writethrough.spec.ts

@@ -0,0 +1,138 @@
+import { DBTestConfiguration } from "../../../tests/extra"
+import {
+  structures,
+  expectFunctionWasCalledTimesWith,
+  mocks,
+} from "../../../tests"
+import { Writethrough } from "../writethrough"
+import { getDB } from "../../db"
+import tk from "timekeeper"
+
+tk.freeze(Date.now())
+
+const DELAY = 5000
+
+describe("writethrough", () => {
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  const db2 = getDB(structures.db.id())
+
+  const writethrough = new Writethrough(db, DELAY)
+  const writethrough2 = new Writethrough(db2, DELAY)
+
+  const docId = structures.uuid()
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  describe("put", () => {
+    let current: any
+
+    it("should be able to store, will go to DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({
+          _id: docId,
+          value: 1,
+        })
+        const output = await db.get<any>(response.id)
+        current = output
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("second put shouldn't update DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ ...current, value: 2 })
+        const output = await db.get<any>(response.id)
+        expect(current._rev).toBe(output._rev)
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("should put it again after delay period", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(Date.now() + DELAY + 1)
+        const response = await writethrough.put({ ...current, value: 3 })
+        const output = await db.get<any>(response.id)
+        expect(response.rev).not.toBe(current._rev)
+        expect(output.value).toBe(3)
+
+        current = output
+      })
+    })
+
+    it("should handle parallel DB updates ignoring conflicts", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(Date.now() + DELAY + 1)
+        const responses = await Promise.all([
+          writethrough.put({ ...current, value: 4 }),
+          writethrough.put({ ...current, value: 4 }),
+          writethrough.put({ ...current, value: 4 }),
+        ])
+
+        // with a lock, this will work
+        const newRev = responses.map(x => x.rev).find(x => x !== current._rev)
+        expect(newRev).toBeDefined()
+        expect(responses.map(x => x.rev)).toEqual(
+          expect.arrayContaining([current._rev, current._rev, newRev])
+        )
+
+        const output = await db.get<any>(current._id)
+        expect(output.value).toBe(4)
+        expect(output._rev).toBe(newRev)
+
+        current = output
+      })
+    })
+
+    it("should handle updates with documents falling behind", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(Date.now() + DELAY + 1)
+
+        const id = structures.uuid()
+        await writethrough.put({ _id: id, value: 1 })
+        const doc = await writethrough.get(id)
+
+        // Updating document
+        tk.freeze(Date.now() + DELAY + 1)
+        await writethrough.put({ ...doc, value: 2 })
+
+        // Update with the old rev value
+        tk.freeze(Date.now() + DELAY + 1)
+        const res = await writethrough.put({
+          ...doc,
+          value: 3,
+        })
+        expect(res.ok).toBe(true)
+
+        const output = await db.get<any>(id)
+        expect(output.value).toBe(3)
+        expect(output._rev).toBe(res.rev)
+      })
+    })
+  })
+
+  describe("get", () => {
+    it("should be able to retrieve", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.get(docId)
+        expect(response.value).toBe(4)
+      })
+    })
+  })
+
+  describe("same doc, different databases (tenancy)", () => {
+    it("should be able to two different databases", async () => {
+      await config.doInTenant(async () => {
+        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
+        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
+        expect(resp1.rev).toBeDefined()
+        expect(resp2.rev).toBeDefined()
+        expect((await db.get<any>("db1")).value).toBe("first")
+        expect((await db2.get<any>("db1")).value).toBe("second")
+      })
+    })
+  })
+})

package/src/cache/user.ts

@@ -0,0 +1,83 @@
+import * as redis from "../redis/init"
+import * as tenancy from "../tenancy"
+import * as context from "../context"
+import * as platform from "../platform"
+import env from "../environment"
+import * as accounts from "../accounts"
+import { UserDB } from "../users"
+import { sdk } from "@budibase/shared-core"
+
+const EXPIRY_SECONDS = 3600
+
+/**
+ * The default populate user function
+ */
+async function populateFromDB(userId: string, tenantId: string) {
+  const db = tenancy.getTenantDB(tenantId)
+  const user = await db.get<any>(userId)
+  user.budibaseAccess = true
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    const account = await accounts.getAccount(user.email)
+    if (account) {
+      user.account = account
+      user.accountPortalAccess = true
+    }
+  }
+
+  return user
+}
+
+/**
+ * Get the requested user by id.
+ * Use redis cache to first read the user.
+ * If not present fallback to loading the user directly and re-caching.
+ * @param {*} userId the id of the user to get
+ * @param {*} tenantId the tenant of the user to get
+ * @param {*} populateUser function to provide the user for re-caching. default to couch db
+ * @returns
+ */
+export async function getUser(
+  userId: string,
+  tenantId?: string,
+  populateUser?: any
+) {
+  if (!populateUser) {
+    populateUser = populateFromDB
+  }
+  if (!tenantId) {
+    try {
+      tenantId = context.getTenantId()
+    } catch (err) {
+      tenantId = await platform.users.lookupTenantId(userId)
+    }
+  }
+  const client = await redis.getUserClient()
+  // try cache
+  let user = await client.get(userId)
+  if (!user) {
+    user = await populateUser(userId, tenantId)
+    await client.store(userId, user, EXPIRY_SECONDS)
+  }
+  if (user && !user.tenantId && tenantId) {
+    // make sure the tenant ID is always correct/set
+    user.tenantId = tenantId
+  }
+  // if has groups, could have builder permissions granted by a group
+  if (user.userGroups && !sdk.users.isGlobalBuilder(user)) {
+    await context.doInTenant(tenantId, async () => {
+      const appIds = await UserDB.getGroupBuilderAppIds(user)
+      if (appIds.length) {
+        const existing = user.builder?.apps || []
+        user.builder = {
+          apps: [...new Set(existing.concat(appIds))],
+        }
+      }
+    })
+  }
+  return user
+}
+
+export async function invalidateUser(userId: string) {
+  const client = await redis.getUserClient()
+  await client.delete(userId)
+}

package/src/cache/writethrough.ts

@@ -0,0 +1,133 @@
+import BaseCache from "./base"
+import { getWritethroughClient } from "../redis/init"
+import { logWarn } from "../logging"
+import { Database, Document, LockName, LockType } from "@budibase/types"
+import * as locks from "../redis/redlockImpl"
+
+const DEFAULT_WRITE_RATE_MS = 10000
+let CACHE: BaseCache | null = null
+
+interface CacheItem {
+  doc: any
+  lastWrite: number
+}
+
+async function getCache() {
+  if (!CACHE) {
+    const client = await getWritethroughClient()
+    CACHE = new BaseCache(client)
+  }
+  return CACHE
+}
+
+function makeCacheKey(db: Database, key: string) {
+  return db.name + key
+}
+
+function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
+  return { doc, lastWrite: lastWrite || Date.now() }
+}
+
+async function put(
+  db: Database,
+  doc: Document,
+  writeRateMs: number = DEFAULT_WRITE_RATE_MS
+) {
+  const cache = await getCache()
+  const key = doc._id
+  let cacheItem: CacheItem | undefined
+  if (key) {
+    cacheItem = await cache.get(makeCacheKey(db, key))
+  }
+  const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs
+  let output = doc
+  if (updateDb) {
+    const lockResponse = await locks.doWithLock(
+      {
+        type: LockType.TRY_ONCE,
+        name: LockName.PERSIST_WRITETHROUGH,
+        resource: key,
+        ttl: 15000,
+      },
+      async () => {
+        const writeDb = async (toWrite: any) => {
+          // doc should contain the _id and _rev
+          const response = await db.put(toWrite, { force: true })
+          output = {
+            ...doc,
+            _id: response.id,
+            _rev: response.rev,
+          }
+        }
+        try {
+          await writeDb(doc)
+        } catch (err: any) {
+          if (err.status !== 409) {
+            throw err
+          } else {
+            // Swallow 409s but log them
+            logWarn(`Ignoring conflict in write-through cache`)
+          }
+        }
+      }
+    )
+
+    if (!lockResponse.executed) {
+      logWarn(`Ignoring redlock conflict in write-through cache`)
+    }
+  }
+  // if we are updating the DB then need to set the lastWrite to now
+  cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite)
+  if (output._id) {
+    await cache.store(makeCacheKey(db, output._id), cacheItem)
+  }
+  return { ok: true, id: output._id, rev: output._rev }
+}
+
+async function get(db: Database, id: string): Promise<any> {
+  const cache = await getCache()
+  const cacheKey = makeCacheKey(db, id)
+  let cacheItem: CacheItem = await cache.get(cacheKey)
+  if (!cacheItem) {
+    const doc = await db.get(id)
+    cacheItem = makeCacheItem(doc)
+    await cache.store(cacheKey, cacheItem)
+  }
+  return cacheItem.doc
+}
+
+async function remove(db: Database, docOrId: any, rev?: any): Promise<void> {
+  const cache = await getCache()
+  if (!docOrId) {
+    throw new Error("No ID/Rev provided.")
+  }
+  const id = typeof docOrId === "string" ? docOrId : docOrId._id
+  rev = typeof docOrId === "string" ? rev : docOrId._rev
+  try {
+    await cache.delete(makeCacheKey(db, id))
+  } finally {
+    await db.remove(id, rev)
+  }
+}
+
+export class Writethrough {
+  db: Database
+  writeRateMs: number
+
+  constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
+    this.db = db
+    this.writeRateMs = writeRateMs
+  }
+
+  async put(doc: any) {
+    return put(this.db, doc, this.writeRateMs)
+  }
+
+  async get(id: string) {
+    return get(this.db, id)
+  }
+
+  async remove(docOrId: any, rev?: any) {
+    return remove(this.db, docOrId, rev)
+  }
+}